xgb 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3c5776d18623264345b660a8c69a88f1ceee3138ff25b8b4378702956e393bb
-  data.tar.gz: a2656e280e6efeefaac2c70a6cdefc54f700c12023d0e00d47967029d75af8b3
+  metadata.gz: ba232bc8a9c27bc9cfb2b7f87afc581acf9a5464df9fccf1b206d912704771fe
+  data.tar.gz: 608ca80f01e81d32a31b796364de14096c89556543fa2892efd86bbf86fac96a
 SHA512:
-  metadata.gz: fbfc62217eee5ed8e3fca644db86908039f1bc40bb6156e8a219baec5ac4230e1fda9f200b3f991cf6683bcd74323e00c50e5c54468bc02e85f2f8399030c410
-  data.tar.gz: 5badc8c673ebae251cef2eaa8e595a1434572737c465e597595c1c323201db01fcc1db862416a6c1a4f87e95ba948dc76ec1851ae3e865fb8c539439340043d9
+  metadata.gz: af38ca270cf3d12a8a7757ff5f32a41f5d68e43f9ad08c585df8efae835d014ddcac51201019aa339143cc081c5c69ef5914d8de06cb4326ccb2864b5b04eba2
+  data.tar.gz: 787bddc1c867d648ffe13c783aede98c62917feca536d1f8fe27e7992e5f2945126ad15b1a4e3ff48cc041723d299cf3cbacdd5557083983575eae22a9583cd8

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.9.0 (2024-10-17)
+
+- Updated XGBoost to 2.1.1
+- Added support for callbacks
+- Added `num_features` and `save_config` methods to `Booster`
+- Added `num_nonmissing` and `data_split_mode` methods to `DMatrix`
+- Dropped support for Ruby < 3.1
+
 ## 0.8.0 (2023-09-13)
 
 - Updated XGBoost to 2.0.0

data/NOTICE.txt

@@ -1,5 +1,5 @@
 Copyright XGBoost contributors
-Copyright 2019-2023 Andrew Kane
+Copyright 2019-2024 Andrew Kane
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -2,7 +2,7 @@
 
 [XGBoost](https://github.com/dmlc/xgboost) - high performance gradient boosting - for Ruby
 
-[![Build Status](https://github.com/ankane/xgboost-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/xgboost-ruby/actions)
+[![Build Status](https://github.com/ankane/xgboost-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/xgboost-ruby/actions)
 
 ## Installation
 

data/lib/xgboost/booster.rb

@@ -1,77 +1,155 @@
 module XGBoost
   class Booster
-    attr_accessor :best_iteration, :feature_names, :feature_types
+    include Utils
+
+    def initialize(params: nil, cache: nil, model_file: nil)
+      cache ||= []
+      cache.each do |d|
+        if !d.is_a?(DMatrix)
+          raise TypeError, "invalid cache item: #{d.class.name}"
+        end
+      end
+
+      dmats = array_of_pointers(cache.map { |d| d.handle })
+      out = ::FFI::MemoryPointer.new(:pointer)
+      check_call FFI.XGBoosterCreate(dmats, cache.length, out)
+      @handle = ::FFI::AutoPointer.new(out.read_pointer, FFI.method(:XGBoosterFree))
 
-    def initialize(params: nil, model_file: nil)
-      @handle = ::FFI::MemoryPointer.new(:pointer)
-      check_result FFI.XGBoosterCreate(nil, 0, @handle)
-      ObjectSpace.define_finalizer(@handle, self.class.finalize(handle_pointer.to_i))
+      cache.each do |d|
+        assign_dmatrix_features(d)
+      end
 
       if model_file
-        check_result FFI.XGBoosterLoadModel(handle_pointer, model_file)
+        check_call FFI.XGBoosterLoadModel(handle, model_file)
       end
 
-      self.best_iteration = 0
       set_param(params)
     end
 
-    def self.finalize(addr)
-      # must use proc instead of stabby lambda
-      proc { FFI.XGBoosterFree(::FFI::Pointer.new(:pointer, addr)) }
+    def [](key_name)
+      if key_name.is_a?(String)
+        return attr(key_name)
+      end
+
+      # TODO slice
+
+      raise TypeError, "expected string"
     end
 
-    def update(dtrain, iteration)
-      check_result FFI.XGBoosterUpdateOneIter(handle_pointer, iteration, dtrain.handle_pointer)
+    def []=(key_name, raw_value)
+      set_attr(**{key_name => raw_value})
     end
 
-    def eval_set(evals, iteration)
-      dmats = array_of_pointers(evals.map { |v| v[0].handle_pointer })
-      evnames = array_of_pointers(evals.map { |v| string_pointer(v[1]) })
+    def save_config
+      length = ::FFI::MemoryPointer.new(:uint64)
+      json_string = ::FFI::MemoryPointer.new(:pointer)
+      check_call FFI.XGBoosterSaveJsonConfig(handle, length, json_string)
+      json_string.read_pointer.read_string(length.read_uint64).force_encoding(Encoding::UTF_8)
+    end
 
-      out_result = ::FFI::MemoryPointer.new(:pointer)
+    def attr(key)
+      ret = ::FFI::MemoryPointer.new(:pointer)
+      success = ::FFI::MemoryPointer.new(:int)
+      check_call FFI.XGBoosterGetAttr(handle, key.to_s, ret, success)
+      success.read_int != 0 ? ret.read_pointer.read_string : nil
+    end
 
-      check_result FFI.XGBoosterEvalOneIter(handle_pointer, iteration, dmats, evnames, evals.size, out_result)
+    def attributes
+      length = ::FFI::MemoryPointer.new(:uint64)
+      sarr = ::FFI::MemoryPointer.new(:pointer)
+      check_call FFI.XGBoosterGetAttrNames(handle, length, sarr)
+      attr_names = from_cstr_to_rbstr(sarr, length)
+      attr_names.to_h { |n| [n, attr(n)] }
+    end
 
-      out_result.read_pointer.read_string
+    def set_attr(**kwargs)
+      kwargs.each do |key, value|
+        check_call FFI.XGBoosterSetAttr(handle, key.to_s, value&.to_s)
+      end
+    end
+
+    def feature_types
+      get_feature_info("feature_type")
+    end
+
+    def feature_types=(features)
+      set_feature_info(features, "feature_type")
+    end
+
+    def feature_names
+      get_feature_info("feature_name")
+    end
+
+    def feature_names=(features)
+      set_feature_info(features, "feature_name")
     end
 
     def set_param(params, value = nil)
       if params.is_a?(Enumerable)
         params.each do |k, v|
-          check_result FFI.XGBoosterSetParam(handle_pointer, k.to_s, v.to_s)
+          check_call FFI.XGBoosterSetParam(handle, k.to_s, v.to_s)
         end
       else
-        check_result FFI.XGBoosterSetParam(handle_pointer, params.to_s, value.to_s)
+        check_call FFI.XGBoosterSetParam(handle, params.to_s, value.to_s)
       end
     end
 
+    def update(dtrain, iteration)
+      check_call FFI.XGBoosterUpdateOneIter(handle, iteration, dtrain.handle)
+    end
+
+    def eval_set(evals, iteration)
+      dmats = array_of_pointers(evals.map { |v| v[0].handle })
+      evnames = array_of_pointers(evals.map { |v| string_pointer(v[1]) })
+
+      out_result = ::FFI::MemoryPointer.new(:pointer)
+
+      check_call FFI.XGBoosterEvalOneIter(handle, iteration, dmats, evnames, evals.size, out_result)
+
+      out_result.read_pointer.read_string
+    end
+
     def predict(data, ntree_limit: nil)
       ntree_limit ||= 0
       out_len = ::FFI::MemoryPointer.new(:uint64)
       out_result = ::FFI::MemoryPointer.new(:pointer)
-      check_result FFI.XGBoosterPredict(handle_pointer, data.handle_pointer, 0, ntree_limit, 0, out_len, out_result)
-      out = out_result.read_pointer.read_array_of_float(read_uint64(out_len))
+      check_call FFI.XGBoosterPredict(handle, data.handle, 0, ntree_limit, 0, out_len, out_result)
+      out = out_result.read_pointer.read_array_of_float(out_len.read_uint64)
       num_class = out.size / data.num_row
       out = out.each_slice(num_class).to_a if num_class > 1
       out
     end
 
     def save_model(fname)
-      check_result FFI.XGBoosterSaveModel(handle_pointer, fname)
+      check_call FFI.XGBoosterSaveModel(handle, fname)
     end
 
-    # returns an array of strings
-    def dump(fmap: "", with_stats: false, dump_format: "text")
-      out_len = ::FFI::MemoryPointer.new(:uint64)
-      out_result = ::FFI::MemoryPointer.new(:pointer)
+    def best_iteration
+      attr(:best_iteration)&.to_i
+    end
 
-      names = feature_names || []
-      fnames = array_of_pointers(names.map { |fname| string_pointer(fname) })
-      ftypes = array_of_pointers(feature_types || Array.new(names.size, string_pointer("float")))
+    def best_iteration=(iteration)
+      set_attr(best_iteration: iteration)
+    end
+
+    def best_score
+      attr(:best_score)&.to_f
+    end
 
-      check_result FFI.XGBoosterDumpModelExWithFeatures(handle_pointer, names.size, fnames, ftypes, with_stats ? 1 : 0, dump_format, out_len, out_result)
+    def best_score=(score)
+      set_attr(best_score: score)
+    end
+
+    def num_boosted_rounds
+      rounds = ::FFI::MemoryPointer.new(:int)
+      check_call FFI.XGBoosterBoostedRounds(handle, rounds)
+      rounds.read_int
+    end
 
-      out_result.read_pointer.get_array_of_string(0, read_uint64(out_len))
+    def num_features
+      features = ::FFI::MemoryPointer.new(:uint64)
+      check_call FFI.XGBoosterGetNumFeature(handle, features)
+      features.read_uint64
     end
 
     def dump_model(fout, fmap: "", with_stats: false, dump_format: "text")
@@ -93,6 +171,20 @@ module XGBoost
       end
     end
 
+    # returns an array of strings
+    def dump(fmap: "", with_stats: false, dump_format: "text")
+      out_len = ::FFI::MemoryPointer.new(:uint64)
+      out_result = ::FFI::MemoryPointer.new(:pointer)
+
+      names = feature_names || []
+      fnames = array_of_pointers(names.map { |fname| string_pointer(fname) })
+      ftypes = array_of_pointers(feature_types || Array.new(names.size, string_pointer("float")))
+
+      check_call FFI.XGBoosterDumpModelExWithFeatures(handle, names.size, fnames, ftypes, with_stats ? 1 : 0, dump_format, out_len, out_result)
+
+      out_result.read_pointer.get_array_of_string(0, out_len.read_uint64)
+    end
+
     def fscore(fmap: "")
       # always weight
       score(fmap: fmap, importance_type: "weight")
@@ -157,48 +249,67 @@ module XGBoost
       end
     end
 
-    def [](key_name)
-      key = string_pointer(key_name)
-      success = ::FFI::MemoryPointer.new(:int)
-      out_result = ::FFI::MemoryPointer.new(:pointer)
-
-      check_result FFI.XGBoosterGetAttr(handle_pointer, key, out_result, success)
-
-      success.read_int == 1 ? out_result.read_pointer.read_string : nil
-    end
-
-    def []=(key_name, raw_value)
-      key = string_pointer(key_name)
-      value = raw_value.nil? ? nil : string_pointer(raw_value)
+    private
 
-      check_result FFI.XGBoosterSetAttr(handle_pointer, key, value)
+    def handle
+      @handle
     end
 
-    def attributes
-      out_len = ::FFI::MemoryPointer.new(:uint64)
-      out_result = ::FFI::MemoryPointer.new(:pointer)
-      check_result FFI.XGBoosterGetAttrNames(handle_pointer, out_len, out_result)
-
-      len = read_uint64(out_len)
-      key_names = len.zero? ? [] : out_result.read_pointer.get_array_of_string(0, len)
-
-      key_names.map { |key_name| [key_name, self[key_name]] }.to_h
-    end
+    def assign_dmatrix_features(data)
+      if data.num_row == 0
+        return
+      end
 
-    private
+      fn = data.feature_names
+      ft = data.feature_types
 
-    def handle_pointer
-      @handle.read_pointer
+      if feature_names.nil?
+        self.feature_names = fn
+      end
+      if feature_types.nil?
+        self.feature_types = ft
+      end
     end
 
-    def array_of_pointers(values)
-      ::FFI::MemoryPointer.new(:pointer, values.size).write_array_of_pointer(values)
+    def get_feature_info(field)
+      length = ::FFI::MemoryPointer.new(:uint64)
+      sarr = ::FFI::MemoryPointer.new(:pointer)
+      if @handle.nil?
+        return nil
+      end
+      check_call(
+        FFI.XGBoosterGetStrFeatureInfo(
+          handle,
+          field,
+          length,
+          sarr
+        )
+      )
+      feature_info = from_cstr_to_rbstr(sarr, length)
+      !feature_info.empty? ? feature_info : nil
     end
 
-    def string_pointer(value)
-      ::FFI::MemoryPointer.from_string(value.to_s)
+    def set_feature_info(features, field)
+      if !features.nil?
+        if !features.is_a?(Array)
+          raise TypeError, "features must be an array"
+        end
+        c_feature_info = array_of_pointers(features.map { |f| string_pointer(f) })
+        check_call(
+          FFI.XGBoosterSetStrFeatureInfo(
+            handle,
+            field,
+            c_feature_info,
+            features.length
+          )
+        )
+      else
+        check_call(
+          FFI.XGBoosterSetStrFeatureInfo(
+            handle, field, nil, 0
+          )
+        )
+      end
     end
-
-    include Utils
   end
 end

data/lib/xgboost/callback_container.rb

@@ -0,0 +1,145 @@
+module XGBoost
+  class CallbackContainer
+    attr_reader :aggregated_cv, :history
+
+    def initialize(callbacks, is_cv: false)
+      @callbacks = callbacks
+      callbacks.each do |callback|
+        unless callback.is_a?(TrainingCallback)
+          raise TypeError, "callback must be an instance of XGBoost::TrainingCallback"
+        end
+      end
+
+      @history = {}
+      @is_cv = is_cv
+    end
+
+    def before_training(model)
+      @callbacks.each do |callback|
+        model = callback.before_training(model)
+        if @is_cv
+          unless model.is_a?(PackedBooster)
+            raise TypeError, "before_training should return the model"
+          end
+        else
+          unless model.is_a?(Booster)
+            raise TypeError, "before_training should return the model"
+          end
+        end
+      end
+      model
+    end
+
+    def after_training(model)
+      @callbacks.each do |callback|
+        model = callback.after_training(model)
+        if @is_cv
+          unless model.is_a?(PackedBooster)
+            raise TypeError, "after_training should return the model"
+          end
+        else
+          unless model.is_a?(Booster)
+            raise TypeError, "after_training should return the model"
+          end
+        end
+      end
+      model
+    end
+
+    def before_iteration(model, epoch, dtrain, evals)
+      @callbacks.any? do |callback|
+        callback.before_iteration(model, epoch, @history)
+      end
+    end
+
+    def after_iteration(model, epoch, dtrain, evals)
+      if @is_cv
+        scores = model.eval_set(epoch)
+        scores = aggcv(scores)
+        @aggregated_cv = scores
+        update_history(scores, epoch)
+      else
+        evals ||= []
+        evals.each do |_, name|
+          if name.include?("-")
+            raise ArgumentError, "Dataset name should not contain `-`"
+          end
+        end
+        score = model.eval_set(evals, epoch)
+        metric_score = parse_eval_str(score)
+        update_history(metric_score, epoch)
+      end
+
+      @callbacks.any? do |callback|
+        callback.after_iteration(model, epoch, @history)
+      end
+    end
+
+    private
+
+    def update_history(score, epoch)
+      score.each do |d|
+        name = d[0]
+        s = d[1]
+        if @is_cv
+          std = d[2]
+          x = [s, std]
+        else
+          x = s
+        end
+        splited_names = name.split("-")
+        data_name = splited_names[0]
+        metric_name = splited_names[1..].join("-")
+        @history[data_name] ||= {}
+        data_history = @history[data_name]
+        data_history[metric_name] ||= []
+        metric_history = data_history[metric_name]
+        metric_history << x
+      end
+    end
+
+    # TODO move
+    def parse_eval_str(result)
+      splited = result.split[1..]
+      # split up `test-error:0.1234`
+      metric_score_str = splited.map { |s| s.split(":") }
+      # convert to float
+      metric_score = metric_score_str.map { |n, s| [n, s.to_f] }
+      metric_score
+    end
+
+    def aggcv(rlist)
+      cvmap = {}
+      idx = rlist[0].split[0]
+      rlist.each do |line|
+        arr = line.split
+        arr[1..].each_with_index do |it, metric_idx|
+          k, v = it.split(":")
+          (cvmap[[metric_idx, k]] ||= []) << v.to_f
+        end
+      end
+      msg = idx
+      results = []
+      cvmap.sort { |x| x[0][0] }.each do |(_, name), s|
+        mean = mean(s)
+        std = stdev(s)
+        results << [name, mean, std]
+      end
+      results
+    end
+
+    def mean(arr)
+      arr.sum / arr.size.to_f
+    end
+
+    # don't subtract one from arr.size
+    def stdev(arr)
+      m = mean(arr)
+      sum = 0
+      arr.each do |v|
+        sum += (v - m) ** 2
+      end
+      Math.sqrt(sum / arr.size)
+    end
+  end
+end

data/lib/xgboost/cv_pack.rb

@@ -0,0 +1,26 @@
+require "forwardable"
+
+module XGBoost
+  class CVPack
+    extend Forwardable
+
+    def_delegators :@bst, :num_boosted_rounds, :best_iteration=, :best_score=
+
+    attr_reader :bst
+
+    def initialize(dtrain, dtest, param)
+      @dtrain = dtrain
+      @dtest = dtest
+      @watchlist = [[dtrain, "train"], [dtest, "test"]]
+      @bst = Booster.new(params: param, cache: [dtrain, dtest])
+    end
+
+    def update(iteration)
+      @bst.update(@dtrain, iteration)
+    end
+
+    def eval_set(iteration)
+      @bst.eval_set(@watchlist, iteration)
+    end
+  end
+end