outliertree 0.1.0

data/NOTICE.txt

@@ -0,0 +1,14 @@
+Copyright (C) 2020 Andrew Kane
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

data/README.md

@@ -0,0 +1,107 @@
+# OutlierTree
+
+:evergreen_tree: [OutlierTree](https://github.com/david-cortes/outliertree) - explainable outlier/anomaly detection - for Ruby
+
+Produces human-readable explanations for why values are detected as outliers
+
+```txt
+Price (2.50) looks low given Department is Books and Sale is false
+```
+
+## Installation
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'outliertree'
+```
+
+## Getting Started
+
+Prep your data
+
+```ruby
+data = [
+  {department: "Books",  sale: false, price: 2.50},
+  {department: "Books",  sale: true,  price: 3.00},
+  {department: "Movies", sale: false, price: 5.00}
+]
+```
+
+Train a model
+
+```ruby
+model = OutlierTree.new
+model.fit(data)
+```
+
+Get outliers
+
+```ruby
+model.outliers(data)
+```
+
+## Parameters
+
+Pass parameters - default values below
+
+```ruby
+OutlierTree.new(
+  max_depth: 4,
+  min_gain: 0.01,
+  z_norm: 2.67,
+  z_outlier: 8.0,
+  pct_outliers: 0.01,
+  min_size_numeric: 25,
+  min_size_categ: 50,
+  categ_split: "binarize",
+  categ_outliers: "tail",
+  numeric_split: "raw",
+  follow_all: false,
+  gain_as_pct: true,
+  nthreads: -1
+)
+```
+
+See a [detailed explanation](https://outliertree.readthedocs.io/en/latest/#outliertree.OutlierTree)
+
+## Data
+
+Data can be an array of hashes
+
+```ruby
+[
+  {department: "Books",  sale: false, price: 2.50},
+  {department: "Books",  sale: true,  price: 3.00},
+  {department: "Movies", sale: false, price: 5.00}
+]
+```
+
+Or a Rover data frame
+
+```ruby
+Rover.read_csv("data.csv")
+```
+
+## History
+
+View the [changelog](https://github.com/ankane/outliertree/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/outliertree/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/outliertree/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone --recursive https://github.com/ankane/outliertree.git
+cd outliertree
+bundle install
+bundle exec rake compile
+bundle exec rake test
+```

data/ext/outliertree/ext.cpp

@@ -0,0 +1,260 @@
+// outliertree
+#include <outlier_tree.hpp>
+
+// rice
+#include <rice/Array.hpp>
+#include <rice/Hash.hpp>
+#include <rice/Module.hpp>
+#include <rice/Object.hpp>
+#include <rice/String.hpp>
+#include <rice/Symbol.hpp>
+
+using Rice::Array;
+using Rice::Hash;
+using Rice::Module;
+using Rice::Object;
+using Rice::String;
+using Rice::Symbol;
+using Rice::define_class_under;
+using Rice::define_module;
+
+template<>
+Object to_ruby<std::vector<char>>(std::vector<char> const & x)
+{
+  Array a;
+  for (size_t i = 0; i < x.size(); i++) {
+    a.push(x[i]);
+  }
+  return a;
+}
+
+template<>
+Object to_ruby<std::vector<int>>(std::vector<int> const & x)
+{
+  Array a;
+  for (size_t i = 0; i < x.size(); i++) {
+    a.push(x[i]);
+  }
+  return a;
+}
+
+template<>
+Object to_ruby<std::vector<unsigned long>>(std::vector<unsigned long> const & x)
+{
+  Array a;
+  for (size_t i = 0; i < x.size(); i++) {
+    a.push(x[i]);
+  }
+  return a;
+}
+
+template<>
+Object to_ruby<std::vector<double>>(std::vector<double> const & x)
+{
+  Array a;
+  for (size_t i = 0; i < x.size(); i++) {
+    a.push(x[i]);
+  }
+  return a;
+}
+
+template<>
+Object to_ruby<ColType>(ColType const & x)
+{
+  switch (x) {
+    case Numeric: return Symbol("numeric");
+    case Categorical: return Symbol("categorical");
+    case Ordinal: return Symbol("ordinal");
+    case NoType: return Symbol("no_type");
+  }
+  throw std::runtime_error("Unknown column type");
+}
+
+template<>
+Object to_ruby<SplitType>(SplitType const & x)
+{
+  switch (x) {
+    case LessOrEqual: return Symbol("less_or_equal");
+    case Greater: return Symbol("greater");
+    case Equal: return Symbol("equal");
+    case NotEqual: return Symbol("not_equal");
+    case InSubset: return Symbol("in_subset");
+    case NotInSubset: return Symbol("not_in_subset");
+    case SingleCateg: return Symbol("single_categ");
+    case SubTrees: return Symbol("sub_trees");
+    case IsNa: return Symbol("is_na");
+    case Root: return Symbol("root");
+  }
+  throw std::runtime_error("Unknown split type");
+}
+
+extern "C"
+void Init_ext()
+{
+  Module rb_mOutlierTree = define_module("OutlierTree");
+  Module rb_mExt = define_module_under(rb_mOutlierTree, "Ext");
+
+  define_class_under<Cluster>(rb_mExt, "Cluster")
+    .define_method("upper_lim", *[](Cluster& self) { return self.upper_lim; })
+    .define_method("display_lim_high", *[](Cluster& self) { return self.display_lim_high; })
+    .define_method("perc_below", *[](Cluster& self) { return self.perc_below; })
+    .define_method("display_lim_low", *[](Cluster& self) { return self.display_lim_low; })
+    .define_method("perc_above", *[](Cluster& self) { return self.perc_above; })
+    .define_method("display_mean", *[](Cluster& self) { return self.display_mean; })
+    .define_method("display_sd", *[](Cluster& self) { return self.display_sd; })
+    .define_method("cluster_size", *[](Cluster& self) { return self.cluster_size; })
+    .define_method("split_point", *[](Cluster& self) { return self.split_point; })
+    .define_method("split_subset", *[](Cluster& self) { return self.split_subset; })
+    .define_method("split_lev", *[](Cluster& self) { return self.split_lev; })
+    .define_method("split_type", *[](Cluster& self) { return self.split_type; })
+    .define_method("column_type", *[](Cluster& self) { return self.column_type; })
+    .define_method("has_na_branch", *[](Cluster& self) { return self.has_NA_branch; })
+    .define_method("col_num", *[](Cluster& self) { return self.col_num; });
+
+  define_class_under<ClusterTree>(rb_mExt, "ClusterTree")
+    .define_method("parent_branch", *[](ClusterTree& self) { return self.parent_branch; })
+    .define_method("parent", *[](ClusterTree& self) { return self.parent; })
+    .define_method("all_branches", *[](ClusterTree& self) { return self.all_branches; })
+    .define_method("column_type", *[](ClusterTree& self) { return self.column_type; })
+    .define_method("col_num", *[](ClusterTree& self) { return self.col_num; })
+    .define_method("split_point", *[](ClusterTree& self) { return self.split_point; })
+    .define_method("split_subset", *[](ClusterTree& self) { return self.split_subset; })
+    .define_method("split_lev", *[](ClusterTree& self) { return self.split_lev; });
+
+  define_class_under<ModelOutputs>(rb_mExt, "ModelOutputs")
+    .define_method("outlier_scores_final", *[](ModelOutputs& self) { return self.outlier_scores_final; })
+    .define_method("outlier_columns_final", *[](ModelOutputs& self) { return self.outlier_columns_final; })
+    .define_method("outlier_clusters_final", *[](ModelOutputs& self) { return self.outlier_clusters_final; })
+    .define_method("outlier_trees_final", *[](ModelOutputs& self) { return self.outlier_trees_final; })
+    .define_method("outlier_depth_final", *[](ModelOutputs& self) { return self.outlier_depth_final; })
+    .define_method("outlier_decimals_distr", *[](ModelOutputs& self) { return self.outlier_decimals_distr; })
+    .define_method("min_decimals_col", *[](ModelOutputs& self) { return self.min_decimals_col; })
+    .define_method(
+      "all_clusters",
+      *[](ModelOutputs& self, size_t i, size_t j) {
+        return self.all_clusters[i][j];
+      })
+    .define_method(
+      "all_trees",
+      *[](ModelOutputs& self, size_t i, size_t j) {
+        return self.all_trees[i][j];
+      });
+
+  rb_mExt
+    .define_singleton_method(
+      "fit_outliers_models",
+      *[](Hash options) {
+        ModelOutputs model_outputs;
+
+        // data
+        size_t nrows = options.get<size_t, Symbol>("nrows");
+        size_t ncols_numeric = options.get<size_t, Symbol>("ncols_numeric");
+        size_t ncols_categ = options.get<size_t, Symbol>("ncols_categ");
+        size_t ncols_ord = options.get<size_t, Symbol>("ncols_ord");
+
+        double *restrict numeric_data = NULL;
+        if (ncols_numeric > 0) {
+          numeric_data = (double*) options.get<String, Symbol>("numeric_data").c_str();
+        }
+
+        int *restrict categorical_data = NULL;
+        int *restrict ncat = NULL;
+        if (ncols_categ > 0) {
+          categorical_data = (int*) options.get<String, Symbol>("categorical_data").c_str();
+          ncat = (int*) options.get<String, Symbol>("ncat").c_str();
+        }
+
+        int *restrict ordinal_data = NULL;
+        int *restrict ncat_ord = NULL;
+        if (ncols_ord > 0) {
+          ordinal_data = (int*) options.get<String, Symbol>("ordinal_data").c_str();
+          ncat_ord = (int*) options.get<String, Symbol>("ncat_ord").c_str();
+        }
+
+        // options
+        char *restrict cols_ignore = NULL;
+        int nthreads = options.get<int, Symbol>("nthreads");
+        bool categ_as_bin = options.get<bool, Symbol>("categ_as_bin");
+        bool ord_as_bin = options.get<bool, Symbol>("ord_as_bin");
+        bool cat_bruteforce_subset = options.get<bool, Symbol>("cat_bruteforce_subset");
+        bool categ_from_maj = options.get<bool, Symbol>("categ_from_maj");
+        bool take_mid = options.get<bool, Symbol>("take_mid");
+        size_t max_depth = options.get<size_t, Symbol>("max_depth");
+        double max_perc_outliers = options.get<double, Symbol>("pct_outliers");
+        size_t min_size_numeric = options.get<size_t, Symbol>("min_size_numeric");
+        size_t min_size_categ = options.get<size_t, Symbol>("min_size_categ");
+        double min_gain = options.get<double, Symbol>("min_gain");
+        bool gain_as_pct = options.get<bool, Symbol>("gain_as_pct");
+        bool follow_all = options.get<bool, Symbol>("follow_all");
+        double z_norm = options.get<double, Symbol>("z_norm");
+        double z_outlier = options.get<double, Symbol>("z_outlier");
+
+        fit_outliers_models(
+          model_outputs,
+          numeric_data,
+          ncols_numeric,
+          categorical_data,
+          ncols_categ,
+          ncat,
+          ordinal_data,
+          ncols_ord,
+          ncat_ord,
+          nrows,
+          cols_ignore,
+          nthreads,
+          categ_as_bin,
+          ord_as_bin,
+          cat_bruteforce_subset,
+          categ_from_maj,
+          take_mid,
+          max_depth,
+          max_perc_outliers,
+          min_size_numeric,
+          min_size_categ,
+          min_gain,
+          gain_as_pct,
+          follow_all,
+          z_norm,
+          z_outlier
+        );
+        return model_outputs;
+      })
+    .define_singleton_method(
+      "find_new_outliers",
+      *[](ModelOutputs& model_outputs, Hash options) {
+        // data
+        size_t nrows = options.get<size_t, Symbol>("nrows");
+        size_t ncols_numeric = options.get<size_t, Symbol>("ncols_numeric");
+        size_t ncols_categ = options.get<size_t, Symbol>("ncols_categ");
+        size_t ncols_ord = options.get<size_t, Symbol>("ncols_ord");
+
+        double *restrict numeric_data = NULL;
+        if (ncols_numeric > 0) {
+          numeric_data = (double*) options.get<String, Symbol>("numeric_data").c_str();
+        }
+
+        int *restrict categorical_data = NULL;
+        if (ncols_categ > 0) {
+          categorical_data = (int*) options.get<String, Symbol>("categorical_data").c_str();
+        }
+
+        int *restrict ordinal_data = NULL;
+        if (ncols_ord > 0) {
+          ordinal_data = (int*) options.get<String, Symbol>("ordinal_data").c_str();
+        }
+
+        // options
+        int nthreads = options.get<int, Symbol>("nthreads");
+
+        find_new_outliers(
+          numeric_data,
+          categorical_data,
+          ordinal_data,
+          nrows,
+          nthreads,
+          model_outputs
+        );
+
+        return model_outputs;
+      });
+}

data/ext/outliertree/extconf.rb

@@ -0,0 +1,21 @@
+require "mkmf-rice"
+
+$CXXFLAGS += " -std=c++11"
+
+apple_clang = RbConfig::CONFIG["CC_VERSION_MESSAGE"] =~ /apple clang/i
+
+# check omp first
+if have_library("omp") || have_library("gomp")
+  $CXXFLAGS += " -Xclang" if apple_clang
+  $CXXFLAGS += " -fopenmp"
+end
+
+ext = File.expand_path(".", __dir__)
+outliertree = File.expand_path("../../vendor/outliertree/src", __dir__)
+
+exclude = %w(Rwrapper.cpp RcppExports.cpp)
+$srcs = Dir["{#{ext},#{outliertree}}/*.{cc,cpp}"].reject { |f| exclude.include?(File.basename(f)) }
+$INCFLAGS += " -I#{outliertree}"
+$VPATH << outliertree
+
+create_makefile("outliertree/ext")

data/lib/outliertree.rb

@@ -0,0 +1,17 @@
+# ext
+require "outliertree/ext"
+
+# stdlib
+require "etc"
+
+# modules
+require "outliertree/dataset"
+require "outliertree/model"
+require "outliertree/result"
+require "outliertree/version"
+
+module OutlierTree
+  def self.new(**options)
+    Model.new(**options)
+  end
+end

data/lib/outliertree/dataset.rb

@@ -0,0 +1,35 @@
+module OutlierTree
+  class Dataset
+    attr_reader :numeric_columns, :categorical_columns
+
+    def initialize(data)
+      @data = data
+
+      if defined?(Rover::DataFrame) && data.is_a?(Rover::DataFrame)
+        @vectors = data.vectors
+        @numeric_columns, @categorical_columns = data.keys.partition { |k, v| ![:object, :bool].include?(data[k].type) }
+      else
+        @vectors = {}
+        raise ArgumentError, "Array elements must be hashes" unless data.all? { |d| d.is_a?(Hash) }
+        keys = data.flat_map(&:keys).uniq
+        keys.each do |k|
+          @vectors[k] = []
+        end
+        data.each do |d|
+          keys.each do |k|
+            @vectors[k] << d[k]
+          end
+        end
+        @numeric_columns, @categorical_columns = keys.partition { |k| @vectors[k].all? { |v| v.nil? || v.is_a?(Numeric) } }
+      end
+    end
+
+    def [](k)
+      @vectors[k]
+    end
+
+    def size
+      @vectors.any? ? @vectors.values.first.size : 0
+    end
+  end
+end