datasketches 0.1.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf7f5b6898a9f014d14fc5fcfa3a6a5da1dd5fc70d1b07bbbdf07a446126d9cd
-  data.tar.gz: 15d5fbb2ee6e0c1a6e7f25a145329eb73da9d7933f5f1475596b9e9c998b31f7
+  metadata.gz: 5c578044053c564421893cc4433f7fe557f23ba9d8a1995fc2a2c5f07742721a
+  data.tar.gz: f4122bd75e19fede015b01a5e5ad8e6130f75babe9c9160cc56f378480a16cee
 SHA512:
-  metadata.gz: e3b412132bdaf7e45f85e1da48690ecccd3c510cc17fd5259f0244cb580f712dc58290fca50a547a9e0952cf7c0548124ed33be9224ba3b83d6dd461d70c2aab
-  data.tar.gz: 5f6d9932824961e11d6f8bd390dbf20e487b6ca8aef6b6440e5f6a11296227382be5261ebaf1da416d5f2354c7c99a97e34871b0964facc724ba33f2b88e96f6
+  metadata.gz: 2d7c4d7306f28356557a816a78033b909561ccd8f843281a2b756e88cbdcb9936da7995ff80871a19e229675ead812aca00d6c639d63a6532998c3c1b35aa953
+  data.tar.gz: fdf0fe1d14e04bfddef9df1ae7958f6571a7f689865aa02e81713d4b250afeeeb8c90a168ce855728d9c831aff6d3ea71df91c71b7269a760c19488c42c92658

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 0.2.2 (2021-07-17)
+
+- Updated DataSketches to 3.1.0
+
+## 0.2.1 (2021-05-23)
+
+- Improved performance
+
+## 0.2.0 (2021-05-17)
+
+- Updated DataSketches to 3.0.0
+- Updated to Rice 4
+
+## 0.1.2 (2021-01-21)
+
+- Added `CpcUnion` and `UpdateThetaSketch`
+
 ## 0.1.1 (2021-01-20)
 
 - Added more sketches

data/README.md

@@ -18,6 +18,7 @@ Distinct counting
 
 - [CPC sketch](#cpc-sketch)
 - [HyperLogLog sketch](#hyperloglog-sketch)
+- [Theta sketch](#theta-sketch)
 
 Most frequent
 
@@ -36,7 +37,7 @@ Sampling
 Create a sketch
 
 ```ruby
-sketch = DataSketches::CpcSketch.new(14)
+sketch = DataSketches::CpcSketch.new
 ```
 
 Add data
@@ -65,6 +66,15 @@ Load a sketch
 sketch = DataSketches::CpcSketch.deserialize(data)
 ```
 
+Get the union
+
+```ruby
+u = DataSketches::CpcUnion.new(14)
+u.update(sketch1)
+u.update(sketch2)
+u.result
+```
+
 ## HyperLogLog Sketch
 
 Create a sketch
@@ -107,7 +117,66 @@ Get the union
 u = DataSketches::HllUnion.new(14)
 u.update(sketch1)
 u.update(sketch2)
-u.estimate
+u.result
+```
+
+## Theta Sketch
+
+Create a sketch
+
+```ruby
+sketch = DataSketches::UpdateThetaSketch.new
+```
+
+Add data
+
+```ruby
+sketch.update(1)
+sketch.update(2.0)
+sketch.update("three")
+```
+
+Estimate the count
+
+```ruby
+sketch.estimate
+```
+
+Save a sketch
+
+```ruby
+data = sketch.serialize
+```
+
+Load a sketch
+
+```ruby
+sketch = DataSketches::UpdateThetaSketch.deserialize(data)
+```
+
+Get the union
+
+```ruby
+u = DataSketches::ThetaUnion.new
+u.update(sketch1)
+u.update(sketch2)
+u.result
+```
+
+Get the intersection
+
+```ruby
+i = DataSketches::ThetaIntersection.new
+i.update(sketch1)
+i.update(sketch2)
+i.result
+```
+
+Compute A not B
+
+```ruby
+d = DataSketches::ThetaANotB.new
+d.compute(a, b)
 ```
 
 ## Frequent Item Sketch
@@ -149,9 +218,9 @@ sketch = DataSketches::FrequentStringsSketch.deserialize(data)
 Create a sketch
 
 ```ruby
-sketch = DataSketches::KllIntsSketch.new(200)
+sketch = DataSketches::KllIntsSketch.new
 # or
-sketch = DataSketches::KllFloatsSketch.new(200)
+sketch = DataSketches::KllFloatsSketch.new
 ```
 
 Add data
@@ -167,6 +236,7 @@ Get quantiles
 ```ruby
 sketch.quantile(0.5)
 sketch.quantiles([0.25, 0.5, 0.75])
+sketch.quantiles(3)
 ```
 
 Get the minimum and maximum values from the stream
@@ -210,6 +280,12 @@ sketch.update(2.0)
 sketch.update("three")
 ```
 
+Sample data
+
+```ruby
+sketch.samples
+```
+
 ## Credits
 
 This library is modeled after the DataSketches [Python API](https://github.com/apache/datasketches-cpp/tree/master/python).

data/ext/datasketches/cpc_wrapper.cpp

@@ -1,36 +1,44 @@
 #include <sstream>
 
 #include <cpc_sketch.hpp>
+#include <cpc_union.hpp>
 
-#include <rice/Constructor.hpp>
-#include <rice/Module.hpp>
+#include "ext.h"
+
+using datasketches::cpc_sketch;
+using datasketches::cpc_union;
+
+using datasketches::CPC_DEFAULT_LG_K;
+using datasketches::DEFAULT_SEED;
+
+using Rice::Arg;
 
 void init_cpc(Rice::Module& m) {
-  Rice::define_class_under<datasketches::cpc_sketch>(m, "CpcSketch")
-    .define_constructor(Rice::Constructor<datasketches::cpc_sketch, uint8_t>())
-    .define_method("lg_k", &datasketches::cpc_sketch::get_lg_k)
-    .define_method("empty?", &datasketches::cpc_sketch::is_empty)
-    .define_method("lower_bound", &datasketches::cpc_sketch::get_lower_bound)
-    .define_method("upper_bound", &datasketches::cpc_sketch::get_upper_bound)
+  Rice::define_class_under<cpc_sketch>(m, "CpcSketch")
+    .define_constructor(Rice::Constructor<cpc_sketch, uint8_t, uint64_t>(), Rice::Arg("lg_k")=CPC_DEFAULT_LG_K, Rice::Arg("seed")=DEFAULT_SEED)
+    .define_method("lg_k", &cpc_sketch::get_lg_k)
+    .define_method("empty?", &cpc_sketch::is_empty)
+    .define_method("lower_bound", &cpc_sketch::get_lower_bound)
+    .define_method("upper_bound", &cpc_sketch::get_upper_bound)
     .define_method(
       "update",
-      *[](datasketches::cpc_sketch& self, Rice::Object datum) {
+      [](cpc_sketch& self, Rice::Object datum) {
         if (FIXNUM_P(datum.value())) {
-          return self.update(from_ruby<int64_t>(datum));
+          return self.update(Rice::detail::From_Ruby<int64_t>().convert(datum));
         } else if (datum.is_a(rb_cNumeric)) {
-          return self.update(from_ruby<double>(datum));
+          return self.update(Rice::detail::From_Ruby<double>().convert(datum));
         } else {
           return self.update(datum.to_s().str());
         }
       })
     .define_method(
       "estimate",
-      *[](datasketches::cpc_sketch& self) {
+      [](cpc_sketch& self) {
         return self.get_estimate();
       })
     .define_method(
       "serialize",
-      *[](datasketches::cpc_sketch& self) {
+      [](cpc_sketch& self) {
         std::ostringstream oss;
         self.serialize(oss);
         return oss.str();
@@ -38,13 +46,22 @@ void init_cpc(Rice::Module& m) {
     // TODO change to summary?
     .define_method(
       "to_string",
-      *[](datasketches::cpc_sketch& self) {
+      [](cpc_sketch& self) {
         return self.to_string();
       })
-    .define_singleton_method(
+    .define_singleton_function(
       "deserialize",
-      *[](std::string& is) {
+      [](const std::string& is) {
         std::istringstream iss(is);
-        return datasketches::cpc_sketch::deserialize(iss);
+        return cpc_sketch::deserialize(iss);
+      });
+
+  Rice::define_class_under<cpc_union>(m, "CpcUnion")
+    .define_constructor(Rice::Constructor<cpc_union, uint8_t, uint64_t>(), Rice::Arg("lg_k"), Rice::Arg("seed")=DEFAULT_SEED)
+    .define_method("result", &cpc_union::get_result)
+    .define_method(
+      "update",
+      [](cpc_union& self, cpc_sketch& sketch) {
+        self.update(sketch);
       });
 }

data/ext/datasketches/ext.cpp

@@ -1,4 +1,4 @@
-#include <rice/Module.hpp>
+#include "ext.h"
 
 void init_cpc(Rice::Module& m);
 void init_fi(Rice::Module& m);

data/ext/datasketches/ext.h

@@ -0,0 +1,4 @@
+#pragma once
+
+#include <rice/rice.hpp>
+#include <rice/stl.hpp>

data/ext/datasketches/extconf.rb

@@ -1,6 +1,6 @@
 require "mkmf-rice"
 
-$CXXFLAGS += " -std=c++11"
+$CXXFLAGS += " -std=c++17 $(optflags)"
 
 ext = File.expand_path(".", __dir__)
 datasketches = File.expand_path("../../vendor/datasketches-cpp", __dir__)

data/ext/datasketches/fi_wrapper.cpp

@@ -2,8 +2,7 @@
 
 #include <frequent_items_sketch.hpp>
 
-#include <rice/Constructor.hpp>
-#include <rice/Module.hpp>
+#include "ext.h"
 
 template<typename T>
 void bind_fi_sketch(Rice::Module& m, const char* name) {
@@ -18,12 +17,12 @@ void bind_fi_sketch(Rice::Module& m, const char* name) {
     .define_method("maximum_error", &datasketches::frequent_items_sketch<T>::get_maximum_error)
     .define_method(
       "update",
-      *[](datasketches::frequent_items_sketch<T>& self, const T item) {
+      [](datasketches::frequent_items_sketch<T>& self, const T item) {
         self.update(item);
       })
     .define_method(
       "serialize",
-      *[](datasketches::frequent_items_sketch<T>& self) {
+      [](datasketches::frequent_items_sketch<T>& self) {
         std::ostringstream oss;
         self.serialize(oss);
         return oss.str();
@@ -31,13 +30,12 @@ void bind_fi_sketch(Rice::Module& m, const char* name) {
     // TODO change to summary?
     .define_method(
       "to_string",
-      *[](datasketches::frequent_items_sketch<T>& self) {
+      [](datasketches::frequent_items_sketch<T>& self) {
         return self.to_string();
       })
-    .define_singleton_method(
+    .define_singleton_function(
       "deserialize",
-      // TODO figure out segfault
-      *[](std::string is) {
+      [](const std::string& is) {
         std::istringstream iss(is);
         return datasketches::frequent_items_sketch<T>::deserialize(iss);
       });

data/ext/datasketches/hll_wrapper.cpp

@@ -2,41 +2,44 @@
 
 #include <hll.hpp>
 
-#include <rice/Constructor.hpp>
-#include <rice/Module.hpp>
+#include "ext.h"
+
+using datasketches::hll_sketch;
+using datasketches::hll_union;
 
 void init_hll(Rice::Module& m) {
-  Rice::define_class_under<datasketches::hll_sketch>(m, "HllSketch")
-    .define_constructor(Rice::Constructor<datasketches::hll_sketch, int>())
-    .define_method("lg_config_k", &datasketches::hll_sketch::get_lg_config_k)
-    .define_method("compact?", &datasketches::hll_sketch::is_compact)
-    .define_method("empty?", &datasketches::hll_sketch::is_empty)
+  Rice::define_class_under<hll_sketch>(m, "HllSketch")
+    .define_constructor(Rice::Constructor<hll_sketch, int>())
+    .define_method("lg_config_k", &hll_sketch::get_lg_config_k)
+    .define_method("compact?", &hll_sketch::is_compact)
+    .define_method("empty?", &hll_sketch::is_empty)
+    .define_method("composite_estimate", &hll_sketch::get_composite_estimate)
     .define_method(
       "update",
-      *[](datasketches::hll_sketch& self, Rice::Object datum) {
+      [](hll_sketch& self, Rice::Object datum) {
         if (FIXNUM_P(datum.value())) {
-          return self.update(from_ruby<int64_t>(datum));
+          return self.update(Rice::detail::From_Ruby<int64_t>().convert(datum));
         } else if (datum.is_a(rb_cNumeric)) {
-          return self.update(from_ruby<double>(datum));
+          return self.update(Rice::detail::From_Ruby<double>().convert(datum));
         } else {
           return self.update(datum.to_s().str());
         }
       })
     .define_method(
       "estimate",
-      *[](datasketches::hll_sketch& self) {
+      [](hll_sketch& self) {
         return self.get_estimate();
       })
     .define_method(
       "serialize_compact",
-      *[](datasketches::hll_sketch& self) {
+      [](hll_sketch& self) {
         std::ostringstream oss;
         self.serialize_compact(oss);
         return oss.str();
       })
     .define_method(
       "serialize_updatable",
-      *[](datasketches::hll_sketch& self) {
+      [](hll_sketch& self) {
         std::ostringstream oss;
         self.serialize_updatable(oss);
         return oss.str();
@@ -44,26 +47,31 @@ void init_hll(Rice::Module& m) {
     // TODO change to summary?
     .define_method(
       "to_string",
-      *[](datasketches::hll_sketch& self) {
+      [](hll_sketch& self) {
         return self.to_string();
       })
-    .define_singleton_method(
+    .define_singleton_function(
       "deserialize",
-      *[](std::string& is) {
+      [](const std::string& is) {
         std::istringstream iss(is);
-        return datasketches::hll_sketch::deserialize(iss);
+        return hll_sketch::deserialize(iss);
       });
 
-  Rice::define_class_under<datasketches::hll_union>(m, "HllUnion")
-    .define_constructor(Rice::Constructor<datasketches::hll_union, int>())
+  Rice::define_class_under<hll_union>(m, "HllUnion")
+    .define_constructor(Rice::Constructor<hll_union, int>())
     .define_method(
       "update",
-      *[](datasketches::hll_union& self, datasketches::hll_sketch& datum) {
-        self.update(datum);
+      [](hll_union& self, hll_sketch& sketch) {
+        self.update(sketch);
       })
     .define_method(
       "estimate",
-      *[](datasketches::hll_union& self) {
+      [](hll_union& self) {
         return self.get_estimate();
+      })
+    .define_method(
+      "result",
+      [](hll_union& self) {
+        return self.get_result();
       });
 }

data/ext/datasketches/kll_wrapper.cpp

@@ -2,67 +2,76 @@
 
 #include <kll_sketch.hpp>
 
-#include <rice/Array.hpp>
-#include <rice/Constructor.hpp>
-#include <rice/Module.hpp>
+#include "ext.h"
 
-template<>
-std::vector<double> from_ruby<std::vector<double>>(Rice::Object x)
-{
-  auto a = Rice::Array(x);
-  std::vector<double> vec(a.size());
-  for (size_t i = 0; i < a.size(); i++) {
-    vec[i] = from_ruby<double>(a[i]);
-  }
-  return vec;
-}
-
-template<>
-Rice::Object to_ruby<std::vector<int>>(std::vector<int> const & x)
-{
-  auto a = Rice::Array();
-  for (size_t i = 0; i < x.size(); i++) {
-    a.push(x[i]);
-  }
-  return a;
-}
+using datasketches::kll_sketch;
 
-template<>
-Rice::Object to_ruby<std::vector<float>>(std::vector<float> const & x)
+namespace Rice::detail
 {
-  auto a = Rice::Array();
-  for (size_t i = 0; i < x.size(); i++) {
-    a.push(x[i]);
-  }
-  return a;
+  template<typename T>
+  class To_Ruby<std::vector<T>>
+  {
+  public:
+    VALUE convert(std::vector<T> const & x)
+    {
+      auto a = rb_ary_new2(x.size());
+      for (const auto& v : x) {
+        detail::protect(rb_ary_push, a, To_Ruby<T>().convert(v));
+      }
+      return a;
+    }
+  };
 }
 
 template<typename T>
 void bind_kll_sketch(Rice::Module& m, const char* name) {
-  Rice::define_class_under<datasketches::kll_sketch<T>>(m, name)
-    .define_constructor(Rice::Constructor<datasketches::kll_sketch<T>, uint16_t>())
-    .define_method("empty?", &datasketches::kll_sketch<T>::is_empty)
-    .define_method("min_value", &datasketches::kll_sketch<T>::get_min_value)
-    .define_method("max_value", &datasketches::kll_sketch<T>::get_max_value)
-    .define_method("quantile", &datasketches::kll_sketch<T>::get_quantile)
+  Rice::define_class_under<kll_sketch<T>>(m, name)
+    .define_constructor(Rice::Constructor<kll_sketch<T>, uint16_t>(), Rice::Arg("k")=kll_sketch<T>::DEFAULT_K)
+    .define_method("empty?", &kll_sketch<T>::is_empty)
+    .define_method("n", &kll_sketch<T>::get_n)
+    .define_method("num_retained", &kll_sketch<T>::get_num_retained)
+    .define_method("estimation_mode?", &kll_sketch<T>::is_estimation_mode)
+    .define_method("min_value", &kll_sketch<T>::get_min_value)
+    .define_method("max_value", &kll_sketch<T>::get_max_value)
+    .define_method("quantile", &kll_sketch<T>::get_quantile)
     .define_method(
       "quantiles",
-      *[](datasketches::kll_sketch<T>& self, std::vector<double> fractions) {
-        return self.get_quantiles(&fractions[0], fractions.size());
+      [](kll_sketch<T>& self, Rice::Object obj) {
+        if (obj.is_a(rb_cArray)) {
+          auto fractions = Rice::detail::From_Ruby<std::vector<double>>().convert(obj);
+          return self.get_quantiles(&fractions[0], fractions.size());
+        } else {
+          return self.get_quantiles(Rice::detail::From_Ruby<size_t>().convert(obj));
+        }
+      })
+    .define_method(
+      "rank",
+      [](kll_sketch<T>& self, const T item) {
+        return self.get_rank(item);
+      })
+    .define_method(
+      "pmf",
+      [](kll_sketch<T>& self, std::vector<T> split_points) {
+        return self.get_PMF(&split_points[0], split_points.size());
+      })
+    .define_method(
+      "cdf",
+      [](kll_sketch<T>& self, std::vector<T> split_points) {
+        return self.get_CDF(&split_points[0], split_points.size());
       })
     .define_method(
       "merge",
-      *[](datasketches::kll_sketch<T>& self, const datasketches::kll_sketch<T>& other) {
+      [](kll_sketch<T>& self, const kll_sketch<T>& other) {
         self.merge(other);
       })
     .define_method(
       "update",
-      *[](datasketches::kll_sketch<T>& self, const T item) {
+      [](kll_sketch<T>& self, const T item) {
         self.update(item);
       })
     .define_method(
       "serialize",
-      *[](datasketches::kll_sketch<T>& self) {
+      [](kll_sketch<T>& self) {
         std::ostringstream oss;
         self.serialize(oss);
         return oss.str();
@@ -70,14 +79,14 @@ void bind_kll_sketch(Rice::Module& m, const char* name) {
     // TODO change to summary?
     .define_method(
       "to_string",
-      *[](datasketches::kll_sketch<T>& self) {
+      [](kll_sketch<T>& self) {
         return self.to_string();
       })
-    .define_singleton_method(
+    .define_singleton_function(
       "deserialize",
-      *[](std::string& is) {
+      [](const std::string& is) {
         std::istringstream iss(is);
-        return datasketches::kll_sketch<T>::deserialize(iss);
+        return kll_sketch<T>::deserialize(iss);
       });
 }