datasketches 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0bf3a004350a15e151c0114a7351519d14f2d9fc793c5ac87a8daa9756be42af
-  data.tar.gz: a48f65a18a1a099e68a4d0de0714b5a1db888c97348c021cd68d6e5a1b066abf
+  metadata.gz: bf7f5b6898a9f014d14fc5fcfa3a6a5da1dd5fc70d1b07bbbdf07a446126d9cd
+  data.tar.gz: 15d5fbb2ee6e0c1a6e7f25a145329eb73da9d7933f5f1475596b9e9c998b31f7
 SHA512:
-  metadata.gz: 89cce97d6a892575dfbe1441d8815742fdd827d26b9f3faafef4e9cc99e5386f025cd00a5e0e1d0188520c2f41405062fcb58c4950d8238d234eff2513234566
-  data.tar.gz: 3cc79a07e80f42b58ea6a3a5e792f72ea79b6f7980076bd92b9cb43c1906bc4c3f30bb6156347c1acce4a2b1d69407ec8178e7775e7ed45bab5930f052fc7b71
+  metadata.gz: e3b412132bdaf7e45f85e1da48690ecccd3c510cc17fd5259f0244cb580f712dc58290fca50a547a9e0952cf7c0548124ed33be9224ba3b83d6dd461d70c2aab
+  data.tar.gz: 5f6d9932824961e11d6f8bd390dbf20e487b6ca8aef6b6440e5f6a11296227382be5261ebaf1da416d5f2354c7c99a97e34871b0964facc724ba33f2b88e96f6

data/CHANGELOG.md

@@ -1,3 +1,8 @@
-## 0.1.0 (unreleased)
+## 0.1.1 (2021-01-20)
+
+- Added more sketches
+- Fixed installation on Linux and Windows
+
+## 0.1.0 (2021-01-19)
 
 - First release

data/README.md

@@ -12,14 +12,26 @@ Add this line to your application’s Gemfile:
 gem 'datasketches'
 ```
 
-## Data Structures
+## Sketch Families
 
 Distinct counting
 
-- [CPC Sketch](#cpc-sketch)
-- [HyperLogLog Sketch](#hyperloglog-sketch)
+- [CPC sketch](#cpc-sketch)
+- [HyperLogLog sketch](#hyperloglog-sketch)
 
-### CPC Sketch
+Most frequent
+
+- [Frequent item sketch](#frequent-item-sketch)
+
+Quantiles and histograms
+
+- [KLL sketch](#kll-sketch)
+
+Sampling
+
+- [VarOpt sketch](#varopt-sketch)
+
+## CPC Sketch
 
 Create a sketch
 
@@ -53,7 +65,7 @@ Load a sketch
 sketch = DataSketches::CpcSketch.deserialize(data)
 ```
 
-### HyperLogLog Sketch
+## HyperLogLog Sketch
 
 Create a sketch
 
@@ -98,6 +110,106 @@ u.update(sketch2)
 u.estimate
 ```
 
+## Frequent Item Sketch
+
+Create a sketch
+
+```ruby
+sketch = DataSketches::FrequentStringsSketch.new(64)
+```
+
+Add data
+
+```ruby
+sketch.update("a")
+sketch.update("b")
+sketch.update("c")
+```
+
+Estimate the frequency of an item
+
+```ruby
+sketch.estimate("a")
+```
+
+Save a sketch
+
+```ruby
+data = sketch.serialize
+```
+
+Load a sketch
+
+```ruby
+sketch = DataSketches::FrequentStringsSketch.deserialize(data)
+```
+
+## KLL Sketch
+
+Create a sketch
+
+```ruby
+sketch = DataSketches::KllIntsSketch.new(200)
+# or
+sketch = DataSketches::KllFloatsSketch.new(200)
+```
+
+Add data
+
+```ruby
+sketch.update(1)
+sketch.update(2)
+sketch.update(3)
+```
+
+Get quantiles
+
+```ruby
+sketch.quantile(0.5)
+sketch.quantiles([0.25, 0.5, 0.75])
+```
+
+Get the minimum and maximum values from the stream
+
+```ruby
+sketch.min_value
+sketch.max_value
+```
+
+Save a sketch
+
+```ruby
+data = sketch.serialize
+```
+
+Load a sketch
+
+```ruby
+sketch = DataSketches::KllIntsSketch.deserialize(data)
+```
+
+Merge sketches
+
+```ruby
+sketch.merge(sketch2)
+```
+
+## VarOpt Sketch
+
+Create a sketch
+
+```ruby
+sketch = DataSketches::VarOptSketch.new(14)
+```
+
+Add data
+
+```ruby
+sketch.update(1)
+sketch.update(2.0)
+sketch.update("three")
+```
+
 ## Credits
 
 This library is modeled after the DataSketches [Python API](https://github.com/apache/datasketches-cpp/tree/master/python).

data/ext/datasketches/cpc_wrapper.cpp

@@ -15,7 +15,7 @@ void init_cpc(Rice::Module& m) {
     .define_method(
       "update",
       *[](datasketches::cpc_sketch& self, Rice::Object datum) {
-        if (datum.is_a(rb_cInteger)) {
+        if (FIXNUM_P(datum.value())) {
           return self.update(from_ruby<int64_t>(datum));
         } else if (datum.is_a(rb_cNumeric)) {
           return self.update(from_ruby<double>(datum));

data/ext/datasketches/ext.cpp

@@ -1,12 +1,20 @@
 #include <rice/Module.hpp>
 
 void init_cpc(Rice::Module& m);
+void init_fi(Rice::Module& m);
 void init_hll(Rice::Module& m);
+void init_kll(Rice::Module& m);
+void init_theta(Rice::Module& m);
+void init_vo(Rice::Module& m);
 
 extern "C"
 void Init_ext()
 {
   Rice::Module m = Rice::define_module("DataSketches");
   init_cpc(m);
+  init_fi(m);
   init_hll(m);
+  init_kll(m);
+  init_theta(m);
+  init_vo(m);
 }

data/ext/datasketches/extconf.rb

@@ -6,6 +6,9 @@ ext = File.expand_path(".", __dir__)
 datasketches = File.expand_path("../../vendor/datasketches-cpp", __dir__)
 
 $srcs = Dir["#{ext}/*.cpp"]
-$INCFLAGS += " -I#{datasketches}/common/include -I#{datasketches}/cpc/include -I#{datasketches}/hll/include"
+
+%w(common cpc fi hll kll sampling theta tuple).each do |v|
+  $INCFLAGS += " -I#{datasketches}/#{v}/include"
+end
 
 create_makefile("datasketches/ext")

data/ext/datasketches/fi_wrapper.cpp

@@ -0,0 +1,48 @@
+#include <sstream>
+
+#include <frequent_items_sketch.hpp>
+
+#include <rice/Constructor.hpp>
+#include <rice/Module.hpp>
+
+template<typename T>
+void bind_fi_sketch(Rice::Module& m, const char* name) {
+  Rice::define_class_under<datasketches::frequent_items_sketch<T>>(m, name)
+    .define_constructor(Rice::Constructor<datasketches::frequent_items_sketch<T>, uint16_t>())
+    .define_method("empty?", &datasketches::frequent_items_sketch<T>::is_empty)
+    .define_method("num_active_items", &datasketches::frequent_items_sketch<T>::get_num_active_items)
+    .define_method("total_weight", &datasketches::frequent_items_sketch<T>::get_total_weight)
+    .define_method("estimate", &datasketches::frequent_items_sketch<T>::get_estimate)
+    .define_method("lower_bound", &datasketches::frequent_items_sketch<T>::get_lower_bound)
+    .define_method("upper_bound", &datasketches::frequent_items_sketch<T>::get_upper_bound)
+    .define_method("maximum_error", &datasketches::frequent_items_sketch<T>::get_maximum_error)
+    .define_method(
+      "update",
+      *[](datasketches::frequent_items_sketch<T>& self, const T item) {
+        self.update(item);
+      })
+    .define_method(
+      "serialize",
+      *[](datasketches::frequent_items_sketch<T>& self) {
+        std::ostringstream oss;
+        self.serialize(oss);
+        return oss.str();
+      })
+    // TODO change to summary?
+    .define_method(
+      "to_string",
+      *[](datasketches::frequent_items_sketch<T>& self) {
+        return self.to_string();
+      })
+    .define_singleton_method(
+      "deserialize",
+      // TODO figure out segfault
+      *[](std::string is) {
+        std::istringstream iss(is);
+        return datasketches::frequent_items_sketch<T>::deserialize(iss);
+      });
+}
+
+void init_fi(Rice::Module& m) {
+  bind_fi_sketch<std::string>(m, "FrequentStringsSketch");
+}

data/ext/datasketches/hll_wrapper.cpp

@@ -14,7 +14,7 @@ void init_hll(Rice::Module& m) {
     .define_method(
       "update",
       *[](datasketches::hll_sketch& self, Rice::Object datum) {
-        if (datum.is_a(rb_cInteger)) {
+        if (FIXNUM_P(datum.value())) {
           return self.update(from_ruby<int64_t>(datum));
         } else if (datum.is_a(rb_cNumeric)) {
           return self.update(from_ruby<double>(datum));
@@ -54,7 +54,7 @@ void init_hll(Rice::Module& m) {
         return datasketches::hll_sketch::deserialize(iss);
       });
 
-  define_class_under<datasketches::hll_union>(m, "HllUnion")
+  Rice::define_class_under<datasketches::hll_union>(m, "HllUnion")
     .define_constructor(Rice::Constructor<datasketches::hll_union, int>())
     .define_method(
       "update",

data/ext/datasketches/kll_wrapper.cpp

@@ -0,0 +1,87 @@
+#include <sstream>
+
+#include <kll_sketch.hpp>
+
+#include <rice/Array.hpp>
+#include <rice/Constructor.hpp>
+#include <rice/Module.hpp>
+
+template<>
+std::vector<double> from_ruby<std::vector<double>>(Rice::Object x)
+{
+  auto a = Rice::Array(x);
+  std::vector<double> vec(a.size());
+  for (size_t i = 0; i < a.size(); i++) {
+    vec[i] = from_ruby<double>(a[i]);
+  }
+  return vec;
+}
+
+template<>
+Rice::Object to_ruby<std::vector<int>>(std::vector<int> const & x)
+{
+  auto a = Rice::Array();
+  for (size_t i = 0; i < x.size(); i++) {
+    a.push(x[i]);
+  }
+  return a;
+}
+
+template<>
+Rice::Object to_ruby<std::vector<float>>(std::vector<float> const & x)
+{
+  auto a = Rice::Array();
+  for (size_t i = 0; i < x.size(); i++) {
+    a.push(x[i]);
+  }
+  return a;
+}
+
+template<typename T>
+void bind_kll_sketch(Rice::Module& m, const char* name) {
+  Rice::define_class_under<datasketches::kll_sketch<T>>(m, name)
+    .define_constructor(Rice::Constructor<datasketches::kll_sketch<T>, uint16_t>())
+    .define_method("empty?", &datasketches::kll_sketch<T>::is_empty)
+    .define_method("min_value", &datasketches::kll_sketch<T>::get_min_value)
+    .define_method("max_value", &datasketches::kll_sketch<T>::get_max_value)
+    .define_method("quantile", &datasketches::kll_sketch<T>::get_quantile)
+    .define_method(
+      "quantiles",
+      *[](datasketches::kll_sketch<T>& self, std::vector<double> fractions) {
+        return self.get_quantiles(&fractions[0], fractions.size());
+      })
+    .define_method(
+      "merge",
+      *[](datasketches::kll_sketch<T>& self, const datasketches::kll_sketch<T>& other) {
+        self.merge(other);
+      })
+    .define_method(
+      "update",
+      *[](datasketches::kll_sketch<T>& self, const T item) {
+        self.update(item);
+      })
+    .define_method(
+      "serialize",
+      *[](datasketches::kll_sketch<T>& self) {
+        std::ostringstream oss;
+        self.serialize(oss);
+        return oss.str();
+      })
+    // TODO change to summary?
+    .define_method(
+      "to_string",
+      *[](datasketches::kll_sketch<T>& self) {
+        return self.to_string();
+      })
+    .define_singleton_method(
+      "deserialize",
+      *[](std::string& is) {
+        std::istringstream iss(is);
+        return datasketches::kll_sketch<T>::deserialize(iss);
+      });
+}
+
+void init_kll(Rice::Module& m) {
+  bind_kll_sketch<int>(m, "KllIntsSketch");
+  bind_kll_sketch<float>(m, "KllFloatsSketch");
+}

data/ext/datasketches/theta_wrapper.cpp

@@ -0,0 +1,12 @@
+#include <sstream>
+
+#include <theta_sketch.hpp>
+
+#include <rice/Constructor.hpp>
+#include <rice/Module.hpp>
+
+void init_theta(Rice::Module& m) {
+  Rice::define_class_under<datasketches::theta_sketch>(m, "ThetaSketch")
+    .define_method("empty?", &datasketches::theta_sketch::is_empty)
+    .define_method("estimate", &datasketches::theta_sketch::get_estimate);
+}

data/ext/datasketches/vo_wrapper.cpp

@@ -0,0 +1,39 @@
+#include <sstream>
+
+#include <var_opt_sketch.hpp>
+
+#include <rice/Array.hpp>
+#include <rice/Constructor.hpp>
+#include <rice/Module.hpp>
+
+template<typename T>
+void bind_vo_sketch(Rice::Module &m, const char* name) {
+  Rice::define_class_under<datasketches::var_opt_sketch<T>>(m, "VarOptSketch")
+    .define_constructor(Rice::Constructor<datasketches::var_opt_sketch<T>, uint32_t>())
+    .define_method("k", &datasketches::var_opt_sketch<T>::get_k)
+    .define_method("n", &datasketches::var_opt_sketch<T>::get_n)
+    .define_method("num_samples", &datasketches::var_opt_sketch<T>::get_num_samples)
+    .define_method("empty?", &datasketches::var_opt_sketch<T>::is_empty)
+    .define_method("reset", &datasketches::var_opt_sketch<T>::reset)
+    .define_method(
+      "samples",
+      *[](datasketches::var_opt_sketch<T>& self) {
+        auto a = Rice::Array();
+        for (auto item : self) {
+          auto t = Rice::Array();
+          t.push(item.first);
+          t.push(item.second);
+          a.push(t);
+        }
+        return a;
+      })
+    .define_method(
+      "update",
+      *[](datasketches::var_opt_sketch<T>& self, const T item) {
+        self.update(item);
+      });
+}
+
+void init_vo(Rice::Module& m) {
+  bind_vo_sketch<Rice::Object>(m, "VarOptSketch");
+}

data/lib/datasketches/version.rb

@@ -1,3 +1,3 @@
 module DataSketches
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datasketches
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Andrew Kane
@@ -38,7 +38,11 @@ files:
 - ext/datasketches/cpc_wrapper.cpp
 - ext/datasketches/ext.cpp
 - ext/datasketches/extconf.rb
+- ext/datasketches/fi_wrapper.cpp
 - ext/datasketches/hll_wrapper.cpp
+- ext/datasketches/kll_wrapper.cpp
+- ext/datasketches/theta_wrapper.cpp
+- ext/datasketches/vo_wrapper.cpp
 - lib/datasketches.rb
 - lib/datasketches/version.rb
 - vendor/datasketches-cpp/CMakeLists.txt