datasketches 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf7f5b6898a9f014d14fc5fcfa3a6a5da1dd5fc70d1b07bbbdf07a446126d9cd
-  data.tar.gz: 15d5fbb2ee6e0c1a6e7f25a145329eb73da9d7933f5f1475596b9e9c998b31f7
+  metadata.gz: 1d74bd4504918fd8217f9cf7d8c24442b5c196b2d874a042f6a2f30a24ae51db
+  data.tar.gz: 7521194ca0d05808b234c6cb1bb5150874c28f7e9536820f728b33b0aec8c52a
 SHA512:
-  metadata.gz: e3b412132bdaf7e45f85e1da48690ecccd3c510cc17fd5259f0244cb580f712dc58290fca50a547a9e0952cf7c0548124ed33be9224ba3b83d6dd461d70c2aab
-  data.tar.gz: 5f6d9932824961e11d6f8bd390dbf20e487b6ca8aef6b6440e5f6a11296227382be5261ebaf1da416d5f2354c7c99a97e34871b0964facc724ba33f2b88e96f6
+  metadata.gz: db41c21f18a51bf285e32556c52e06453810448d1d75d11cd1022576a98d5069977d65fe6bab7e5bd6425468531414fdd801a1bde1bfdf82fc05f455d97b80fd
+  data.tar.gz: 892e9f2c4b891194cfc1af8387494e78eb2e703b4a56f7c91e1b4784f0bd2439f35f8e25f92e954be44418d2964e804f892ab687df2acf8e9b64d43e0049cca0

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.2 (2021-01-21)
+
+- Added `CpcUnion` and `UpdateThetaSketch`
+
 ## 0.1.1 (2021-01-20)
 
 - Added more sketches

data/README.md

@@ -18,6 +18,7 @@ Distinct counting
 
 - [CPC sketch](#cpc-sketch)
 - [HyperLogLog sketch](#hyperloglog-sketch)
+- [Theta sketch](#theta-sketch)
 
 Most frequent
 
@@ -36,7 +37,7 @@ Sampling
 Create a sketch
 
 ```ruby
-sketch = DataSketches::CpcSketch.new(14)
+sketch = DataSketches::CpcSketch.new
 ```
 
 Add data
@@ -65,6 +66,15 @@ Load a sketch
 sketch = DataSketches::CpcSketch.deserialize(data)
 ```
 
+Get the union
+
+```ruby
+u = DataSketches::CpcUnion.new(14)
+u.update(sketch1)
+u.update(sketch2)
+u.result
+```
+
 ## HyperLogLog Sketch
 
 Create a sketch
@@ -107,7 +117,66 @@ Get the union
 u = DataSketches::HllUnion.new(14)
 u.update(sketch1)
 u.update(sketch2)
-u.estimate
+u.result
+```
+
+## Theta Sketch
+
+Create a sketch
+
+```ruby
+sketch = DataSketches::UpdateThetaSketch.new
+```
+
+Add data
+
+```ruby
+sketch.update(1)
+sketch.update(2.0)
+sketch.update("three")
+```
+
+Estimate the count
+
+```ruby
+sketch.estimate
+```
+
+Save a sketch
+
+```ruby
+data = sketch.serialize
+```
+
+Load a sketch
+
+```ruby
+sketch = DataSketches::UpdateThetaSketch.deserialize(data)
+```
+
+Get the union
+
+```ruby
+u = DataSketches::ThetaUnion.new
+u.update(sketch1)
+u.update(sketch2)
+u.result
+```
+
+Get the intersection
+
+```ruby
+i = DataSketches::ThetaIntersection.new
+i.update(sketch1)
+i.update(sketch2)
+i.result
+```
+
+Compute A not B
+
+```ruby
+d = DataSketches::ThetaANotB.new
+d.compute(a, b)
 ```
 
 ## Frequent Item Sketch
@@ -149,9 +218,9 @@ sketch = DataSketches::FrequentStringsSketch.deserialize(data)
 Create a sketch
 
 ```ruby
-sketch = DataSketches::KllIntsSketch.new(200)
+sketch = DataSketches::KllIntsSketch.new
 # or
-sketch = DataSketches::KllFloatsSketch.new(200)
+sketch = DataSketches::KllFloatsSketch.new
 ```
 
 Add data
@@ -167,6 +236,7 @@ Get quantiles
 ```ruby
 sketch.quantile(0.5)
 sketch.quantiles([0.25, 0.5, 0.75])
+sketch.quantiles(3)
 ```
 
 Get the minimum and maximum values from the stream
@@ -210,6 +280,12 @@ sketch.update(2.0)
 sketch.update("three")
 ```
 
+Sample data
+
+```ruby
+sketch.samples
+```
+
 ## Credits
 
 This library is modeled after the DataSketches [Python API](https://github.com/apache/datasketches-cpp/tree/master/python).

data/ext/datasketches/cpc_wrapper.cpp

@@ -1,20 +1,29 @@
 #include <sstream>
 
 #include <cpc_sketch.hpp>
+#include <cpc_union.hpp>
 
 #include <rice/Constructor.hpp>
 #include <rice/Module.hpp>
 
+using datasketches::cpc_sketch;
+using datasketches::cpc_union;
+
+using datasketches::CPC_DEFAULT_LG_K;
+using datasketches::DEFAULT_SEED;
+
+using Rice::Arg;
+
 void init_cpc(Rice::Module& m) {
-  Rice::define_class_under<datasketches::cpc_sketch>(m, "CpcSketch")
-    .define_constructor(Rice::Constructor<datasketches::cpc_sketch, uint8_t>())
-    .define_method("lg_k", &datasketches::cpc_sketch::get_lg_k)
-    .define_method("empty?", &datasketches::cpc_sketch::is_empty)
-    .define_method("lower_bound", &datasketches::cpc_sketch::get_lower_bound)
-    .define_method("upper_bound", &datasketches::cpc_sketch::get_upper_bound)
+  Rice::define_class_under<cpc_sketch>(m, "CpcSketch")
+    .define_constructor(Rice::Constructor<cpc_sketch, uint8_t, uint64_t>(), (Rice::Arg("lg_k")=CPC_DEFAULT_LG_K, Rice::Arg("seed")=DEFAULT_SEED))
+    .define_method("lg_k", &cpc_sketch::get_lg_k)
+    .define_method("empty?", &cpc_sketch::is_empty)
+    .define_method("lower_bound", &cpc_sketch::get_lower_bound)
+    .define_method("upper_bound", &cpc_sketch::get_upper_bound)
     .define_method(
       "update",
-      *[](datasketches::cpc_sketch& self, Rice::Object datum) {
+      *[](cpc_sketch& self, Rice::Object datum) {
         if (FIXNUM_P(datum.value())) {
           return self.update(from_ruby<int64_t>(datum));
         } else if (datum.is_a(rb_cNumeric)) {
@@ -25,12 +34,12 @@ void init_cpc(Rice::Module& m) {
       })
     .define_method(
       "estimate",
-      *[](datasketches::cpc_sketch& self) {
+      *[](cpc_sketch& self) {
         return self.get_estimate();
       })
     .define_method(
       "serialize",
-      *[](datasketches::cpc_sketch& self) {
+      *[](cpc_sketch& self) {
         std::ostringstream oss;
         self.serialize(oss);
         return oss.str();
@@ -38,13 +47,22 @@ void init_cpc(Rice::Module& m) {
     // TODO change to summary?
     .define_method(
       "to_string",
-      *[](datasketches::cpc_sketch& self) {
+      *[](cpc_sketch& self) {
         return self.to_string();
       })
     .define_singleton_method(
       "deserialize",
       *[](std::string& is) {
         std::istringstream iss(is);
-        return datasketches::cpc_sketch::deserialize(iss);
+        return cpc_sketch::deserialize(iss);
+      });
+
+  Rice::define_class_under<cpc_union>(m, "CpcUnion")
+    .define_constructor(Rice::Constructor<cpc_union, uint8_t, uint64_t>(), (Rice::Arg("lg_k"), Rice::Arg("seed")=DEFAULT_SEED))
+    .define_method("result", &cpc_union::get_result)
+    .define_method(
+      "update",
+      *[](cpc_union& self, cpc_sketch& sketch) {
+        self.update(sketch);
       });
 }

data/ext/datasketches/hll_wrapper.cpp

@@ -5,15 +5,19 @@
 #include <rice/Constructor.hpp>
 #include <rice/Module.hpp>
 
+using datasketches::hll_sketch;
+using datasketches::hll_union;
+
 void init_hll(Rice::Module& m) {
-  Rice::define_class_under<datasketches::hll_sketch>(m, "HllSketch")
-    .define_constructor(Rice::Constructor<datasketches::hll_sketch, int>())
-    .define_method("lg_config_k", &datasketches::hll_sketch::get_lg_config_k)
-    .define_method("compact?", &datasketches::hll_sketch::is_compact)
-    .define_method("empty?", &datasketches::hll_sketch::is_empty)
+  Rice::define_class_under<hll_sketch>(m, "HllSketch")
+    .define_constructor(Rice::Constructor<hll_sketch, int>())
+    .define_method("lg_config_k", &hll_sketch::get_lg_config_k)
+    .define_method("compact?", &hll_sketch::is_compact)
+    .define_method("empty?", &hll_sketch::is_empty)
+    .define_method("composite_estimate", &hll_sketch::get_composite_estimate)
     .define_method(
       "update",
-      *[](datasketches::hll_sketch& self, Rice::Object datum) {
+      *[](hll_sketch& self, Rice::Object datum) {
         if (FIXNUM_P(datum.value())) {
           return self.update(from_ruby<int64_t>(datum));
         } else if (datum.is_a(rb_cNumeric)) {
@@ -24,19 +28,19 @@ void init_hll(Rice::Module& m) {
       })
     .define_method(
       "estimate",
-      *[](datasketches::hll_sketch& self) {
+      *[](hll_sketch& self) {
         return self.get_estimate();
       })
     .define_method(
       "serialize_compact",
-      *[](datasketches::hll_sketch& self) {
+      *[](hll_sketch& self) {
         std::ostringstream oss;
         self.serialize_compact(oss);
         return oss.str();
       })
     .define_method(
       "serialize_updatable",
-      *[](datasketches::hll_sketch& self) {
+      *[](hll_sketch& self) {
         std::ostringstream oss;
         self.serialize_updatable(oss);
         return oss.str();
@@ -44,26 +48,31 @@ void init_hll(Rice::Module& m) {
     // TODO change to summary?
     .define_method(
       "to_string",
-      *[](datasketches::hll_sketch& self) {
+      *[](hll_sketch& self) {
         return self.to_string();
       })
     .define_singleton_method(
       "deserialize",
       *[](std::string& is) {
         std::istringstream iss(is);
-        return datasketches::hll_sketch::deserialize(iss);
+        return hll_sketch::deserialize(iss);
       });
 
-  Rice::define_class_under<datasketches::hll_union>(m, "HllUnion")
-    .define_constructor(Rice::Constructor<datasketches::hll_union, int>())
+  Rice::define_class_under<hll_union>(m, "HllUnion")
+    .define_constructor(Rice::Constructor<hll_union, int>())
     .define_method(
       "update",
-      *[](datasketches::hll_union& self, datasketches::hll_sketch& datum) {
-        self.update(datum);
+      *[](hll_union& self, hll_sketch& sketch) {
+        self.update(sketch);
       })
     .define_method(
       "estimate",
-      *[](datasketches::hll_union& self) {
+      *[](hll_union& self) {
         return self.get_estimate();
+      })
+    .define_method(
+      "result",
+      *[](hll_union& self) {
+        return self.get_result();
       });
 }

data/ext/datasketches/kll_wrapper.cpp

@@ -6,12 +6,36 @@
 #include <rice/Constructor.hpp>
 #include <rice/Module.hpp>
 
+using datasketches::kll_sketch;
+
+template<>
+std::vector<int> from_ruby<std::vector<int>>(Rice::Object x)
+{
+  auto a = Rice::Array(x);
+  std::vector<int> vec(a.size());
+  for (long i = 0; i < a.size(); i++) {
+    vec[i] = from_ruby<int>(a[i]);
+  }
+  return vec;
+}
+
+template<>
+std::vector<float> from_ruby<std::vector<float>>(Rice::Object x)
+{
+  auto a = Rice::Array(x);
+  std::vector<float> vec(a.size());
+  for (long i = 0; i < a.size(); i++) {
+    vec[i] = from_ruby<float>(a[i]);
+  }
+  return vec;
+}
+
 template<>
 std::vector<double> from_ruby<std::vector<double>>(Rice::Object x)
 {
   auto a = Rice::Array(x);
   std::vector<double> vec(a.size());
-  for (size_t i = 0; i < a.size(); i++) {
+  for (long i = 0; i < a.size(); i++) {
     vec[i] = from_ruby<double>(a[i]);
   }
   return vec;
@@ -37,32 +61,65 @@ Rice::Object to_ruby<std::vector<float>>(std::vector<float> const & x)
   return a;
 }
 
+template<>
+Rice::Object to_ruby<std::vector<double>>(std::vector<double> const & x)
+{
+  auto a = Rice::Array();
+  for (size_t i = 0; i < x.size(); i++) {
+    a.push(x[i]);
+  }
+  return a;
+}
+
 template<typename T>
 void bind_kll_sketch(Rice::Module& m, const char* name) {
-  Rice::define_class_under<datasketches::kll_sketch<T>>(m, name)
-    .define_constructor(Rice::Constructor<datasketches::kll_sketch<T>, uint16_t>())
-    .define_method("empty?", &datasketches::kll_sketch<T>::is_empty)
-    .define_method("min_value", &datasketches::kll_sketch<T>::get_min_value)
-    .define_method("max_value", &datasketches::kll_sketch<T>::get_max_value)
-    .define_method("quantile", &datasketches::kll_sketch<T>::get_quantile)
+  Rice::define_class_under<kll_sketch<T>>(m, name)
+    .define_constructor(Rice::Constructor<kll_sketch<T>, uint16_t>(), (Rice::Arg("k")=kll_sketch<T>::DEFAULT_K))
+    .define_method("empty?", &kll_sketch<T>::is_empty)
+    .define_method("n", &kll_sketch<T>::get_n)
+    .define_method("num_retained", &kll_sketch<T>::get_num_retained)
+    .define_method("estimation_mode?", &kll_sketch<T>::is_estimation_mode)
+    .define_method("min_value", &kll_sketch<T>::get_min_value)
+    .define_method("max_value", &kll_sketch<T>::get_max_value)
+    .define_method("quantile", &kll_sketch<T>::get_quantile)
     .define_method(
       "quantiles",
-      *[](datasketches::kll_sketch<T>& self, std::vector<double> fractions) {
-        return self.get_quantiles(&fractions[0], fractions.size());
+      *[](kll_sketch<T>& self, Rice::Object obj) {
+        if (obj.is_a(rb_cArray)) {
+          auto fractions = from_ruby<std::vector<double>>(obj);
+          return self.get_quantiles(&fractions[0], fractions.size());
+        } else {
+          return self.get_quantiles(from_ruby<size_t>(obj));
+        }
+      })
+    .define_method(
+      "rank",
+      *[](kll_sketch<T>& self, const T item) {
+        return self.get_rank(item);
+      })
+    .define_method(
+      "pmf",
+      *[](kll_sketch<T>& self, std::vector<T> split_points) {
+        return self.get_PMF(&split_points[0], split_points.size());
+      })
+    .define_method(
+      "cdf",
+      *[](kll_sketch<T>& self, std::vector<T> split_points) {
+        return self.get_CDF(&split_points[0], split_points.size());
       })
     .define_method(
       "merge",
-      *[](datasketches::kll_sketch<T>& self, const datasketches::kll_sketch<T>& other) {
+      *[](kll_sketch<T>& self, const kll_sketch<T>& other) {
         self.merge(other);
       })
     .define_method(
       "update",
-      *[](datasketches::kll_sketch<T>& self, const T item) {
+      *[](kll_sketch<T>& self, const T item) {
         self.update(item);
       })
     .define_method(
       "serialize",
-      *[](datasketches::kll_sketch<T>& self) {
+      *[](kll_sketch<T>& self) {
         std::ostringstream oss;
         self.serialize(oss);
         return oss.str();
@@ -70,14 +127,14 @@ void bind_kll_sketch(Rice::Module& m, const char* name) {
     // TODO change to summary?
     .define_method(
       "to_string",
-      *[](datasketches::kll_sketch<T>& self) {
+      *[](kll_sketch<T>& self) {
         return self.to_string();
       })
     .define_singleton_method(
       "deserialize",
       *[](std::string& is) {
         std::istringstream iss(is);
-        return datasketches::kll_sketch<T>::deserialize(iss);
+        return kll_sketch<T>::deserialize(iss);
       });
 }
 

data/ext/datasketches/theta_wrapper.cpp

@@ -1,12 +1,102 @@
 #include <sstream>
 
 #include <theta_sketch.hpp>
+#include <theta_union.hpp>
+#include <theta_intersection.hpp>
+#include <theta_a_not_b.hpp>
 
 #include <rice/Constructor.hpp>
 #include <rice/Module.hpp>
 
+using datasketches::theta_sketch;
+using datasketches::update_theta_sketch;
+using datasketches::compact_theta_sketch;
+using datasketches::theta_union;
+using datasketches::theta_intersection;
+using datasketches::theta_a_not_b;
+
+using datasketches::DEFAULT_SEED;
+
+using Rice::Arg;
+
 void init_theta(Rice::Module& m) {
-  Rice::define_class_under<datasketches::theta_sketch>(m, "ThetaSketch")
-    .define_method("empty?", &datasketches::theta_sketch::is_empty)
-    .define_method("estimate", &datasketches::theta_sketch::get_estimate);
+  Rice::define_class_under<theta_sketch>(m, "ThetaSketch")
+    .define_method("empty?", &theta_sketch::is_empty)
+    .define_method("estimate", &theta_sketch::get_estimate)
+    .define_method("lower_bound", &theta_sketch::get_lower_bound)
+    .define_method("upper_bound", &theta_sketch::get_upper_bound)
+    .define_method(
+      "serialize",
+      *[](theta_sketch& self) {
+        std::ostringstream oss;
+        self.serialize(oss);
+        return oss.str();
+      });
+
+  Rice::define_class_under<update_theta_sketch, theta_sketch>(m, "UpdateThetaSketch")
+    .define_singleton_method(
+      "new",
+      *[](uint8_t lg_k, double p, uint64_t seed) {
+        update_theta_sketch::builder builder;
+        builder.set_lg_k(lg_k);
+        builder.set_p(p);
+        builder.set_seed(seed);
+        return builder.build();
+      },
+      (Arg("lg_k")=update_theta_sketch::builder::DEFAULT_LG_K, Arg("p")=1.0, Arg("seed")=DEFAULT_SEED))
+    .define_method("compact", &update_theta_sketch::compact, (Arg("ordered")=true))
+    .define_method(
+      "update",
+      *[](update_theta_sketch& self, Rice::Object datum) {
+        if (FIXNUM_P(datum.value())) {
+          return self.update(from_ruby<int64_t>(datum));
+        } else if (datum.is_a(rb_cNumeric)) {
+          return self.update(from_ruby<double>(datum));
+        } else {
+          return self.update(datum.to_s().str());
+        }
+      })
+    .define_method(
+      "estimate",
+      *[](update_theta_sketch& self) {
+        return self.get_estimate();
+      })
+    .define_singleton_method(
+      "deserialize",
+      *[](std::string& is) {
+        std::istringstream iss(is);
+        return update_theta_sketch::deserialize(iss);
+      });
+
+  Rice::define_class_under<compact_theta_sketch, theta_sketch>(m, "CompactThetaSketch")
+    .define_singleton_method(
+      "deserialize",
+      *[](std::string& is) {
+        std::istringstream iss(is);
+        return compact_theta_sketch::deserialize(iss);
+      });
+
+  Rice::define_class_under<theta_union>(m, "ThetaUnion")
+    .define_singleton_method(
+      "new",
+      *[](uint8_t lg_k, double p, uint64_t seed) {
+        theta_union::builder builder;
+        builder.set_lg_k(lg_k);
+        builder.set_p(p);
+        builder.set_seed(seed);
+        return builder.build();
+      },
+      (Arg("lg_k")=update_theta_sketch::builder::DEFAULT_LG_K, Arg("p")=1.0, Arg("seed")=DEFAULT_SEED))
+    .define_method("update", &theta_union::update)
+    .define_method("result", &theta_union::get_result, (Arg("ordered")=true));
+
+  Rice::define_class_under<theta_intersection>(m, "ThetaIntersection")
+    .define_constructor(Rice::Constructor<theta_intersection, uint64_t>(), (Arg("seed")=DEFAULT_SEED))
+    .define_method("update", &theta_intersection::update)
+    .define_method("result", &theta_intersection::get_result, (Arg("ordered")=true))
+    .define_method("result?", &theta_intersection::has_result);
+
+  Rice::define_class_under<theta_a_not_b>(m, "ThetaANotB")
+    .define_constructor(Rice::Constructor<theta_a_not_b, uint64_t>(), (Arg("seed")=DEFAULT_SEED))
+    .define_method("compute", &theta_a_not_b::compute, (Arg("a"), Arg("b"), Arg("ordered")=true));
 }

data/ext/datasketches/vo_wrapper.cpp

@@ -6,18 +6,20 @@
 #include <rice/Constructor.hpp>
 #include <rice/Module.hpp>
 
+using datasketches::var_opt_sketch;
+
 template<typename T>
 void bind_vo_sketch(Rice::Module &m, const char* name) {
-  Rice::define_class_under<datasketches::var_opt_sketch<T>>(m, "VarOptSketch")
-    .define_constructor(Rice::Constructor<datasketches::var_opt_sketch<T>, uint32_t>())
-    .define_method("k", &datasketches::var_opt_sketch<T>::get_k)
-    .define_method("n", &datasketches::var_opt_sketch<T>::get_n)
-    .define_method("num_samples", &datasketches::var_opt_sketch<T>::get_num_samples)
-    .define_method("empty?", &datasketches::var_opt_sketch<T>::is_empty)
-    .define_method("reset", &datasketches::var_opt_sketch<T>::reset)
+  Rice::define_class_under<var_opt_sketch<T>>(m, "VarOptSketch")
+    .define_constructor(Rice::Constructor<var_opt_sketch<T>, uint32_t>())
+    .define_method("k", &var_opt_sketch<T>::get_k)
+    .define_method("n", &var_opt_sketch<T>::get_n)
+    .define_method("num_samples", &var_opt_sketch<T>::get_num_samples)
+    .define_method("empty?", &var_opt_sketch<T>::is_empty)
+    .define_method("reset", &var_opt_sketch<T>::reset)
     .define_method(
       "samples",
-      *[](datasketches::var_opt_sketch<T>& self) {
+      *[](var_opt_sketch<T>& self) {
         auto a = Rice::Array();
         for (auto item : self) {
           auto t = Rice::Array();
@@ -29,7 +31,7 @@ void bind_vo_sketch(Rice::Module &m, const char* name) {
       })
     .define_method(
       "update",
-      *[](datasketches::var_opt_sketch<T>& self, const T item) {
+      *[](var_opt_sketch<T>& self, const T item) {
         self.update(item);
       });
 }

data/lib/datasketches/version.rb

@@ -1,3 +1,3 @@
 module DataSketches
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datasketches
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-20 00:00:00.000000000 Z
+date: 2021-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice