isotree 0.1.1 → 0.2.0

data/ext/isotree/extconf.rb

@@ -1,7 +1,6 @@
 require "mkmf-rice"
 
-# TODO add -D_ENABLE_CEREAL
-$CXXFLAGS += " -std=c++11 -D_USE_MERSENNE_TWISTER"
+$CXXFLAGS += " -std=c++17 -D_USE_MERSENNE_TWISTER -D_ENABLE_CEREAL"
 
 apple_clang = RbConfig::CONFIG["CC_VERSION_MESSAGE"] =~ /apple clang/i
 
@@ -13,10 +12,11 @@ end
 
 ext = File.expand_path(".", __dir__)
 isotree = File.expand_path("../../vendor/isotree/src", __dir__)
+cereal = File.expand_path("../../vendor/cereal/include", __dir__)
 
 exclude = %w(Rwrapper.cpp RcppExports.cpp)
 $srcs = Dir["{#{ext},#{isotree}}/*.{cc,cpp}"].reject { |f| exclude.include?(File.basename(f)) }
-$INCFLAGS << " -I#{isotree}"
+$INCFLAGS << " -I#{isotree} -I#{cereal}"
 $VPATH << isotree
 
 create_makefile("isotree/ext")

data/lib/isotree.rb

@@ -3,7 +3,9 @@ require "isotree/ext"
 
 # stdlib
 require "etc"
+require "json"
 
 # modules
+require "isotree/dataset"
 require "isotree/isolation_forest"
 require "isotree/version"

data/lib/isotree/dataset.rb

@@ -0,0 +1,73 @@
+module IsoTree
+  class Dataset
+    attr_reader :numeric_columns, :categorical_columns, :array_type
+
+    def initialize(data)
+      @data = data
+
+      if defined?(Rover::DataFrame) && data.is_a?(Rover::DataFrame)
+        @vectors = data.vectors
+        @numeric_columns, @categorical_columns = data.keys.partition { |k, v| ![:object, :bool].include?(data[k].type) }
+        @array_type = false
+      elsif defined?(Numo::NArray) && data.is_a?(Numo::NArray)
+        raise ArgumentError, "Input must have 2 dimensions" if data.ndim != 2
+
+        data = data.cast_to(Numo::DFloat)
+        ncols = data.shape[1]
+
+        @numeric_columns = ncols.times.to_a
+        @categorical_columns = []
+
+        @vectors = {}
+        @numeric_columns.each do |k|
+          @vectors[k] = data[true, k]
+        end
+        @array_type = true
+      else
+        data = data.to_a
+
+        hashes = data.all? { |d| d.is_a?(Hash) }
+        arrays = !hashes && data.all? { |d| d.is_a?(Array) }
+        unless hashes || arrays
+          raise ArgumentError, "Array elements must be all hashes or arrays"
+        end
+
+        nrows = data.size
+        ncols = data.first ? data.first.size : 0
+        if data.any? { |r| r.size != ncols }
+          raise ArgumentError, "All rows must have the same number of columns"
+        end
+
+        keys =
+          if hashes
+            data.flat_map(&:keys).uniq
+          else
+            ncols.times.to_a
+          end
+
+        @vectors = {}
+        keys.each do |k|
+          @vectors[k] = []
+        end
+        data.each do |d|
+          keys.each do |k|
+            @vectors[k] << d[k]
+          end
+        end
+
+        @numeric_columns, @categorical_columns = keys.partition { |k| @vectors[k].all? { |v| v.nil? || v.is_a?(Numeric) } }
+        @array_type = arrays
+      end
+
+      raise ArgumentError, "No data" if size == 0
+    end
+
+    def [](k)
+      @vectors[k]
+    end
+
+    def size
+      @vectors.any? ? @vectors.values.first.size : 0
+    end
+  end
+end

data/lib/isotree/isolation_forest.rb

@@ -4,9 +4,11 @@ module IsoTree
       sample_size: nil, ntrees: 500, ndim: 3, ntry: 3,
       prob_pick_avg_gain: 0, prob_pick_pooled_gain: 0,
       prob_split_avg_gain: 0, prob_split_pooled_gain: 0,
-      min_gain: 0, all_perm: false, coef_by_prop: false,
+      min_gain: 0, missing_action: "impute", new_categ_action: "smallest",
+      categ_split_type: "subset", all_perm: false, coef_by_prop: false,
       sample_with_replacement: false, penalize_range: true,
-      weigh_by_kurtosis: false, min_imp_obs: 3, random_seed: 1, nthreads: -1
+      weigh_by_kurtosis: false, coefs: "normal", min_imp_obs: 3, depth_imp: "higher",
+      weigh_imp_rows: "inverse", random_seed: 1, nthreads: -1
     )
 
       @sample_size = sample_size
@@ -18,12 +20,18 @@ module IsoTree
       @prob_split_avg_gain = prob_split_avg_gain
       @prob_split_pooled_gain = prob_split_pooled_gain
       @min_gain = min_gain
+      @missing_action = missing_action
+      @new_categ_action = new_categ_action
+      @categ_split_type = categ_split_type
       @all_perm = all_perm
       @coef_by_prop = coef_by_prop
       @sample_with_replacement = sample_with_replacement
       @penalize_range = penalize_range
       @weigh_by_kurtosis = weigh_by_kurtosis
+      @coefs = coefs
       @min_imp_obs = min_imp_obs
+      @depth_imp = depth_imp
+      @weigh_imp_rows = weigh_imp_rows
       @random_seed = random_seed
 
       # etc module returns virtual cores
@@ -32,57 +40,202 @@ module IsoTree
     end
 
     def fit(x)
+      x = Dataset.new(x)
+      prep_fit(x)
       options = data_options(x).merge(fit_options)
       options[:sample_size] ||= options[:nrows]
-      @ncols = options[:ncols]
       @ext_iso_forest = Ext.fit_iforest(options)
     end
 
-    def predict(x)
-      raise "Not fit" unless @ext_iso_forest
+    def predict(x, output: "score")
+      check_fit
+
+      x = Dataset.new(x)
+      prep_predict(x)
+
       options = data_options(x).merge(nthreads: @nthreads)
-      if options[:ncols] != @ncols
-        raise ArgumentError, "Input must have #{@ncols} columns for this model"
+      case output
+      when "score"
+        options[:standardize] = true
+      when "avg_depth"
+        options[:standardize] = false
+      else
+        raise ArgumentError, "Unknown output"
       end
+
       Ext.predict_iforest(@ext_iso_forest, options)
     end
 
+    # same format as Python so models are compatible
+    def export_model(path)
+      check_fit
+
+      File.write("#{path}.metadata", JSON.pretty_generate(export_metadata))
+      Ext.serialize_ext_isoforest(@ext_iso_forest, path)
+    end
+
+    def self.import_model(path)
+      model = new
+      metadata = JSON.parse(File.read("#{path}.metadata"))
+      model.send(:import_metadata, metadata)
+      model.instance_variable_set(:@ext_iso_forest, Ext.deserialize_ext_isoforest(path))
+      model
+    end
+
     private
 
-    # TODO support categorical data
-    def data_options(x)
-      if defined?(Numo::NArray) && x.is_a?(Numo::NArray)
-        raise ArgumentError, "Input must have 2 dimensions" if x.ndim != 2
-        x = x.cast_to(Numo::DFloat)
-        numeric_data = x.to_binary
-        nrows, ncols = x.shape
-      else
-        x = x.to_a
-        nrows = x.size
-        ncols = x.first ? x.first.size : 0
-        if x.any? { |r| r.size != ncols }
-          raise ArgumentError, "All rows must have the same number of columns"
-        end
-        numeric_data = x.flatten(1).pack("d*")
+    def export_metadata
+      data_info = {
+        ncols_numeric: @numeric_columns.size,
+        ncols_categ: @categorical_columns.size,
+        cols_numeric: @numeric_columns,
+        cols_categ: @categorical_columns,
+        cat_levels: @categorical_columns.map { |v| @categories[v].keys }
+      }
+
+      # Ruby-specific
+      data_info[:sym_numeric] = @numeric_columns.map { |v| v.is_a?(Symbol) }
+      data_info[:sym_categ] = @categorical_columns.map { |v| v.is_a?(Symbol) }
+
+      model_info = {
+        ndim: @ndim,
+        nthreads: @nthreads,
+        build_imputer: false
+      }
+
+      params = {}
+      PARAM_KEYS.each do |k|
+        params[k] = instance_variable_get("@#{k}")
       end
-      raise ArgumentError, "No data" if nrows == 0
 
       {
-        nrows: nrows,
-        ncols: ncols,
-        numeric_data: numeric_data
+        data_info: data_info,
+        model_info: model_info,
+        params: params
       }
     end
 
+    def import_metadata(metadata)
+      data_info = metadata["data_info"]
+      model_info = metadata["model_info"]
+      params = metadata["params"]
+
+      # Ruby-specific
+      sym_numeric = data_info["sym_numeric"].to_a
+      sym_categ = data_info["sym_categ"].to_a
+
+      @numeric_columns = data_info["cols_numeric"].map.with_index { |v, i| sym_numeric[i] ? v.to_sym : v }
+      @categorical_columns = data_info["cols_categ"].map.with_index { |v, i| sym_categ[i] ? v.to_sym : v }
+      @categories = {}
+      @categorical_columns.zip(data_info["cat_levels"]) do |col, levels|
+        @categories[col] = levels.map.with_index.to_h
+      end
+
+      @ndim = model_info["ndim"]
+      @nthreads = model_info["nthreads"]
+
+      PARAM_KEYS.each do |k|
+        instance_variable_set("@#{k}", params[k.to_s])
+      end
+    end
+
+    def check_fit
+      raise "Not fit" unless @ext_iso_forest
+    end
+
+    def prep_fit(df)
+      @numeric_columns = df.numeric_columns
+      @categorical_columns = df.categorical_columns
+      @categories = {}
+      @categorical_columns.each do |k|
+        @categories[k] = df[k].uniq.to_a.compact.map.with_index.to_h
+      end
+    end
+
+    # TODO handle column type mismatches
+    def prep_predict(df)
+      expected_columns = @numeric_columns + @categorical_columns
+      if df.array_type
+        if df.numeric_columns.size + df.categorical_columns.size != expected_columns.size
+          raise ArgumentError, "Input must have #{expected_columns.size} columns for this model"
+        end
+      end
+      expected_columns.each do |k|
+        raise ArgumentError, "Missing column: #{k}" unless df[k]
+      end
+    end
+
+    def data_options(df)
+      options = {}
+
+      # numeric
+      numeric_data = String.new
+      @numeric_columns.each do |k|
+        v = df[k]
+        v = v.to_numo if v.respond_to?(:to_numo) # Rover
+        binary_str =
+          if v.respond_to?(:to_binary) # Rover and Numo
+            v.cast_to(Numo::DFloat).to_binary
+          else
+            v.pack("d*")
+          end
+        numeric_data << binary_str
+      end
+      options[:numeric_data] = numeric_data
+      options[:ncols_numeric] = @numeric_columns.size
+
+      # categorical
+      categorical_data = String.new
+      ncat = String.new
+      @categorical_columns.each do |k|
+        categories = @categories[k]
+        # for unseen values, set to categories.size
+        categories_size = categories.size
+        values = df[k].map { |v| v.nil? ? -1 : (categories[v] || categories_size) }
+        # TODO make more efficient
+        if values.any? { |v| v == categories_size }
+          warn "[isotree] Unseen values in column: #{k}"
+        end
+
+        v = values
+        v = v.to_numo if v.respond_to?(:to_numo) # Rover
+        binary_str =
+          if v.respond_to?(:to_binary) # Rover and Numo
+            v.cast_to(Numo::Int32).to_binary
+          else
+            v.pack("i*")
+          end
+        categorical_data << binary_str
+        ncat << [categories.size].pack("i")
+      end
+      options[:categorical_data] = categorical_data
+      options[:ncols_categ] = @categorical_columns.size
+      options[:ncat] = ncat
+
+      options[:nrows] = df.size
+      options
+    end
+
+    PARAM_KEYS = %i(
+      sample_size ntrees ntry max_depth
+      prob_pick_avg_gain prob_pick_pooled_gain
+      prob_split_avg_gain prob_split_pooled_gain min_gain
+      missing_action new_categ_action categ_split_type coefs depth_imp
+      weigh_imp_rows min_imp_obs random_seed all_perm coef_by_prop
+      weights_as_sample_prob sample_with_replacement penalize_range
+      weigh_by_kurtosis assume_full_distr
+    )
+
     def fit_options
       keys = %i(
         sample_size ntrees ndim ntry
         prob_pick_avg_gain prob_pick_pooled_gain
         prob_split_avg_gain prob_split_pooled_gain
-        min_gain all_perm coef_by_prop
+        min_gain missing_action new_categ_action
+        categ_split_type all_perm coef_by_prop
         sample_with_replacement penalize_range
-        weigh_by_kurtosis min_imp_obs
-        random_seed nthreads
+        weigh_by_kurtosis coefs min_imp_obs depth_imp
+        weigh_imp_rows random_seed nthreads
       )
       options = {}
       keys.each do |k|

data/lib/isotree/version.rb

@@ -1,3 +1,3 @@
 module IsoTree
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/vendor/cereal/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2014, Randolph Voorhies, Shane Grant
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of cereal nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL RANDOLPH VOORHIES OR SHANE GRANT BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/vendor/cereal/README.md

@@ -0,0 +1,85 @@
+cereal - A C++11 library for serialization
+==========================================
+
+<img src="http://uscilab.github.io/cereal/assets/img/cerealboxside.png" align="right"/><p>cereal is a header-only C++11 serialization library.  cereal takes arbitrary data types and reversibly turns them into different representations, such as compact binary encodings, XML, or JSON.  cereal was designed to be fast, light-weight, and easy to extend - it has no external dependencies and can be easily bundled with other code or used standalone.</p>
+
+### cereal has great documentation
+
+Looking for more information on how cereal works and its documentation?  Visit [cereal's web page](http://USCiLab.github.com/cereal) to get the latest information.
+
+### cereal is easy to use
+
+Installation and use of of cereal is fully documented on the [main web page](http://USCiLab.github.com/cereal), but this is a quick and dirty version:
+
+* Download cereal and place the headers somewhere your code can see them
+* Write serialization functions for your custom types or use the built in support for the standard library cereal provides
+* Use the serialization archives to load and save data
+
+```cpp
+#include <cereal/types/unordered_map.hpp>
+#include <cereal/types/memory.hpp>
+#include <cereal/archives/binary.hpp>
+#include <fstream>
+    
+struct MyRecord
+{
+  uint8_t x, y;
+  float z;
+  
+  template <class Archive>
+  void serialize( Archive & ar )
+  {
+    ar( x, y, z );
+  }
+};
+    
+struct SomeData
+{
+  int32_t id;
+  std::shared_ptr<std::unordered_map<uint32_t, MyRecord>> data;
+  
+  template <class Archive>
+  void save( Archive & ar ) const
+  {
+    ar( data );
+  }
+      
+  template <class Archive>
+  void load( Archive & ar )
+  {
+    static int32_t idGen = 0;
+    id = idGen++;
+    ar( data );
+  }
+};
+
+int main()
+{
+  std::ofstream os("out.cereal", std::ios::binary);
+  cereal::BinaryOutputArchive archive( os );
+
+  SomeData myData;
+  archive( myData );
+
+  return 0;
+}
+```    
+
+### cereal has a mailing list
+
+Either get in touch over <a href="mailto:cerealcpp@googlegroups.com">email</a> or [on the web](https://groups.google.com/forum/#!forum/cerealcpp).
+
+
+
+## cereal has a permissive license
+
+cereal is licensed under the [BSD license](http://opensource.org/licenses/BSD-3-Clause).
+
+## cereal build status
+
+* master : [![Build Status](https://travis-ci.com/USCiLab/cereal.svg?branch=master)](https://travis-ci.com/USCiLab/cereal)
+[![Build status](https://ci.appveyor.com/api/projects/status/91aou6smj36or0vb/branch/master?svg=true)](https://ci.appveyor.com/project/AzothAmmo/cereal/branch/master)
+
+---
+
+Were you looking for the Haskell cereal?  Go <a href="https://github.com/GaloisInc/cereal">here</a>.