datasketches 0.1.2 → 0.2.0

data/vendor/datasketches-cpp/kll/test/kll_sketch_custom_type_test.cpp

@@ -26,7 +26,8 @@
 
 namespace datasketches {
 
-typedef kll_sketch<test_type, test_type_less, test_type_serde, test_allocator<test_type>> kll_test_type_sketch;
+using kll_test_type_sketch = kll_sketch<test_type, test_type_less, test_type_serde, test_allocator<test_type>>;
+using alloc = test_allocator<test_type>;
 
 TEST_CASE("kll sketch custom type", "[kll_sketch]") {
 
@@ -34,7 +35,7 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
   test_allocator_total_bytes = 0;
 
   SECTION("compact level zero") {
-    kll_test_type_sketch sketch(8);
+    kll_test_type_sketch sketch(8, 0);
     REQUIRE_THROWS_AS(sketch.get_quantile(0), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_min_value(), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_max_value(), std::runtime_error);
@@ -59,10 +60,10 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
   }
 
   SECTION("merge small") {
-    kll_test_type_sketch sketch1(8);
+    kll_test_type_sketch sketch1(8, 0);
     sketch1.update(1);
 
-    kll_test_type_sketch sketch2(8);
+    kll_test_type_sketch sketch2(8, 0);
     sketch2.update(2);
 
     sketch2.merge(sketch1);
@@ -76,7 +77,7 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
   }
 
   SECTION("merge higher levels") {
-    kll_test_type_sketch sketch1(8);
+    kll_test_type_sketch sketch1(8, 0);
     sketch1.update(1);
     sketch1.update(2);
     sketch1.update(3);
@@ -87,7 +88,7 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
     sketch1.update(8);
     sketch1.update(9);
 
-    kll_test_type_sketch sketch2(8);
+    kll_test_type_sketch sketch2(8, 0);
     sketch2.update(10);
     sketch2.update(11);
     sketch2.update(12);
@@ -109,7 +110,7 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
   }
 
   SECTION("serialize deserialize") {
-    kll_test_type_sketch sketch1;
+    kll_test_type_sketch sketch1(200, 0);
 
     const int n = 1000;
     for (int i = 0; i < n; i++) sketch1.update(i);
@@ -117,7 +118,7 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch1.serialize(s);
     REQUIRE((size_t) s.tellp() == sketch1.get_serialized_size_bytes());
-    auto sketch2 = kll_test_type_sketch::deserialize(s);
+    auto sketch2 = kll_test_type_sketch::deserialize(s, alloc(0));
     REQUIRE((size_t) s.tellg() == sketch2.get_serialized_size_bytes());
     REQUIRE(s.tellg() == s.tellp());
     REQUIRE(sketch2.is_empty() == sketch1.is_empty());
@@ -135,9 +136,9 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
   }
 
   SECTION("moving merge") {
-    kll_test_type_sketch sketch1(8);
+    kll_test_type_sketch sketch1(8, 0);
     for (int i = 0; i < 10; i++) sketch1.update(i);
-    kll_test_type_sketch sketch2(8);
+    kll_test_type_sketch sketch2(8, 0);
     sketch2.update(10);
     sketch2.merge(std::move(sketch1));
     REQUIRE(sketch2.get_min_value().get_value() == 0);

data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp

@@ -48,14 +48,14 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   test_allocator_total_bytes = 0;
 
   SECTION("k limits") {
-    kll_float_sketch sketch1(kll_float_sketch::MIN_K); // this should work
-    kll_float_sketch sketch2(kll_float_sketch::MAX_K); // this should work
-    REQUIRE_THROWS_AS(new kll_float_sketch(kll_float_sketch::MIN_K - 1), std::invalid_argument);
+    kll_float_sketch sketch1(kll_float_sketch::MIN_K, 0); // this should work
+    kll_float_sketch sketch2(kll_float_sketch::MAX_K, 0); // this should work
+    REQUIRE_THROWS_AS(new kll_float_sketch(kll_float_sketch::MIN_K - 1, 0), std::invalid_argument);
     // MAX_K + 1 makes no sense because k is uint16_t
   }
 
   SECTION("empty") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     REQUIRE(sketch.is_empty());
     REQUIRE_FALSE(sketch.is_estimation_mode());
     REQUIRE(sketch.get_n() == 0);
@@ -79,13 +79,13 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 }
 
   SECTION("get bad quantile") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     sketch.update(0); // has to be non-empty to reach the check
     REQUIRE_THROWS_AS(sketch.get_quantile(-1), std::invalid_argument);
   }
 
   SECTION("one item") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     sketch.update(1);
     REQUIRE_FALSE(sketch.is_empty());
     REQUIRE_FALSE(sketch.is_estimation_mode());
@@ -112,7 +112,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("NaN") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     sketch.update(std::numeric_limits<float>::quiet_NaN());
     REQUIRE(sketch.is_empty());
 
@@ -122,7 +122,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("many items, exact mode") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     const uint32_t n(200);
     for (uint32_t i = 0; i < n; i++) {
       sketch.update(i);
@@ -157,7 +157,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("10 items") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     sketch.update(1);
     sketch.update(2);
     sketch.update(3);
@@ -175,7 +175,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("100 items") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     for (int i = 0; i < 100; ++i) sketch.update(i);
     REQUIRE(sketch.get_quantile(0) == 0);
     REQUIRE(sketch.get_quantile(0.01) == 1);
@@ -185,7 +185,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("many items, estimation mode") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     const int n(1000000);
     for (int i = 0; i < n; i++) {
       sketch.update(i);
@@ -227,7 +227,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("consistency between get_rank adn get_PMF/CDF") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     const int n = 1000;
     float values[n];
     for (int i = 0; i < n; i++) {
@@ -256,7 +256,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     std::ifstream is;
     is.exceptions(std::ios::failbit | std::ios::badbit);
     is.open(testBinaryInputPath + "kll_sketch_from_java.sk", std::ios::binary);
-    auto sketch = kll_float_sketch::deserialize(is);
+    auto sketch = kll_float_sketch::deserialize(is, test_allocator<float>(0));
     REQUIRE_FALSE(sketch.is_empty());
     REQUIRE(sketch.is_estimation_mode());
     REQUIRE(sketch.get_n() == 1000000);
@@ -266,11 +266,11 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("stream serialize deserialize empty") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(s);
+    auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
@@ -283,9 +283,9 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("bytes serialize deserialize empty") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     auto bytes = sketch.serialize();
-    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size());
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
     REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
@@ -298,12 +298,12 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("serialize deserialize one item") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     sketch.update(1);
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(s);
+    auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
     REQUIRE(s.tellg() == s.tellp());
     REQUIRE_FALSE(sketch2.is_empty());
@@ -321,7 +321,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     std::ifstream is;
     is.exceptions(std::ios::failbit | std::ios::badbit);
     is.open(testBinaryInputPath + "kll_sketch_float_one_item_v1.sk", std::ios::binary);
-    auto sketch = kll_float_sketch::deserialize(is);
+    auto sketch = kll_float_sketch::deserialize(is, test_allocator<float>(0));
     REQUIRE_FALSE(sketch.is_empty());
     REQUIRE_FALSE(sketch.is_estimation_mode());
     REQUIRE(sketch.get_n() == 1);
@@ -331,13 +331,13 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("stream serialize deserialize many floats") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     const int n(1000);
     for (int i = 0; i < n; i++) sketch.update(i);
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(s);
+    auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
     REQUIRE(s.tellg() == s.tellp());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
@@ -354,12 +354,12 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("bytes serialize deserialize many floats") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     const int n(1000);
     for (int i = 0; i < n; i++) sketch.update(i);
     auto bytes = sketch.serialize();
     REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size());
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
@@ -414,7 +414,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("out of order split points, float") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     sketch.update(0); // has too be non-empty to reach the check
     float split_points[2] = {1, 0};
     REQUIRE_THROWS_AS(sketch.get_CDF(split_points, 2), std::invalid_argument);
@@ -428,15 +428,15 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("NaN split point") {
-    kll_float_sketch sketch;
+    kll_float_sketch sketch(200, 0);
     sketch.update(0); // has too be non-empty to reach the check
     float split_points[1] = {std::numeric_limits<float>::quiet_NaN()};
     REQUIRE_THROWS_AS(sketch.get_CDF(split_points, 1), std::invalid_argument);
   }
 
   SECTION("merge") {
-    kll_float_sketch sketch1;
-    kll_float_sketch sketch2;
+    kll_float_sketch sketch1(200, 0);
+    kll_float_sketch sketch2(200, 0);
     const int n = 10000;
     for (int i = 0; i < n; i++) {
       sketch1.update(i);
@@ -458,8 +458,8 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("merge lower k") {
-    kll_float_sketch sketch1(256);
-    kll_float_sketch sketch2(128);
+    kll_float_sketch sketch1(256, 0);
+    kll_float_sketch sketch2(128, 0);
     const int n = 10000;
     for (int i = 0; i < n; i++) {
       sketch1.update(i);
@@ -471,6 +471,9 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch2.get_min_value() == n);
     REQUIRE(sketch2.get_max_value() == 2.0f * n - 1);
 
+    REQUIRE(sketch1.get_k() == 256);
+    REQUIRE(sketch2.get_k() == 128);
+
     REQUIRE(sketch1.get_normalized_rank_error(false) < sketch2.get_normalized_rank_error(false));
     REQUIRE(sketch1.get_normalized_rank_error(true) < sketch2.get_normalized_rank_error(true));
 
@@ -488,8 +491,8 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("merge exact mode, lower k") {
-    kll_float_sketch sketch1(256);
-    kll_float_sketch sketch2(128);
+    kll_float_sketch sketch1(256, 0);
+    kll_float_sketch sketch2(128, 0);
     const int n = 10000;
     for (int i = 0; i < n; i++) {
       sketch1.update(i);
@@ -513,8 +516,8 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("merge min value from other") {
-    kll_float_sketch sketch1;
-    kll_float_sketch sketch2;
+    kll_float_sketch sketch1(200, 0);
+    kll_float_sketch sketch2(200, 0);
     sketch1.update(1);
     sketch2.update(2);
     sketch2.merge(sketch1);
@@ -523,9 +526,9 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("merge min and max values from other") {
-    kll_float_sketch sketch1;
+    kll_float_sketch sketch1(200, 0);
     for (int i = 0; i < 1000000; i++) sketch1.update(i);
-    kll_float_sketch sketch2;
+    kll_float_sketch sketch2(200, 0);
     sketch2.merge(sketch1);
     REQUIRE(sketch2.get_min_value() == 0.0f);
     REQUIRE(sketch2.get_max_value() == 999999.0f);
@@ -560,7 +563,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("sketch of strings stream") {
-    kll_string_sketch sketch1;
+    kll_string_sketch sketch1(200, 0);
     REQUIRE_THROWS_AS(sketch1.get_quantile(0), std::runtime_error);
     REQUIRE_THROWS_AS(sketch1.get_min_value(), std::runtime_error);
     REQUIRE_THROWS_AS(sketch1.get_max_value(), std::runtime_error);
@@ -575,7 +578,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch1.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch1.get_serialized_size_bytes());
-    auto sketch2 = kll_string_sketch::deserialize(s);
+    auto sketch2 = kll_string_sketch::deserialize(s, test_allocator<std::string>(0));
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
     REQUIRE(s.tellg() == s.tellp());
     REQUIRE(sketch2.is_empty() == sketch1.is_empty());
@@ -599,7 +602,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   }
 
   SECTION("sketch of strings bytes") {
-    kll_string_sketch sketch1;
+    kll_string_sketch sketch1(200, 0);
     REQUIRE_THROWS_AS(sketch1.get_quantile(0), std::runtime_error);
     REQUIRE_THROWS_AS(sketch1.get_min_value(), std::runtime_error);
     REQUIRE_THROWS_AS(sketch1.get_max_value(), std::runtime_error);
@@ -613,7 +616,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
     auto bytes = sketch1.serialize();
     REQUIRE(bytes.size() == sketch1.get_serialized_size_bytes());
-    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size());
+    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
     REQUIRE(sketch2.is_empty() == sketch1.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch1.is_estimation_mode());
@@ -630,11 +633,11 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
 
   SECTION("sketch of strings, single item, bytes") {
-    kll_string_sketch sketch1;
+    kll_string_sketch sketch1(200, 0);
     sketch1.update("a");
     auto bytes = sketch1.serialize();
     REQUIRE(bytes.size() == sketch1.get_serialized_size_bytes());
-    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size());
+    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
   }
 

data/vendor/datasketches-cpp/python/CMakeLists.txt

@@ -35,6 +35,7 @@ target_link_libraries(python
     fi
     theta
     sampling
+    req
     pybind11::module
 )
 
@@ -57,5 +58,6 @@ target_sources(python
     src/fi_wrapper.cpp
     src/theta_wrapper.cpp
     src/vo_wrapper.cpp
+    src/req_wrapper.cpp
     src/vector_of_kll.cpp
 )

data/vendor/datasketches-cpp/python/README.md

@@ -1,4 +1,4 @@
-# Python Wrapper for Datasketches
+# Python Wrapper for Apache DataSketches
 
 ## Installation
 
@@ -39,13 +39,16 @@ tox
 
 ## Usage
 
-Having installed the library, loading the Datasketches library in Python is simple: `import datasketches`.
+Having installed the library, loading the Apache Datasketches library in Python is simple: `import datasketches`.
 
 ## Available Sketch Classes
 
-- KLL
+- KLL (Absolute Error Quantiles)
     - `kll_ints_sketch`
     - `kll_floats_sketch`
+- REQ (Relative Error Quantiles)
+    - `req_ints_sketch`
+    - `req_floats_sketch`
 - Frequent Items
     - `frequent_strings_sketch`
     - Error types are `frequent_items_error_type.{NO_FALSE_NEGATIVES | NO_FALSE_POSITIVES}`

data/vendor/datasketches-cpp/python/src/datasketches.cpp

@@ -27,6 +27,7 @@ void init_fi(py::module& m);
 void init_cpc(py::module& m);
 void init_theta(py::module& m);
 void init_vo(py::module& m);
+void init_req(py::module& m);
 void init_vector_of_kll(py::module& m);
 
 PYBIND11_MODULE(datasketches, m) {
@@ -36,5 +37,6 @@ PYBIND11_MODULE(datasketches, m) {
   init_cpc(m);
   init_theta(m);
   init_vo(m);
+  init_req(m);
   init_vector_of_kll(m);
 }

data/vendor/datasketches-cpp/python/src/hll_wrapper.cpp

@@ -113,8 +113,6 @@ void init_hll(py::module &m) {
          "Returns the approximate lower error bound given the specified number of standard deviations in {1, 2, 3}")
     .def("get_upper_bound", &hll_union::get_upper_bound, py::arg("num_std_devs"),
          "Returns the approximate upper error bound given the specified number of standard deviations in {1, 2, 3}")
-    .def("is_compact", &hll_union::is_compact,
-         "True if the union is compact, otherwise False")
     .def("is_empty", &hll_union::is_empty,
          "True if the union is empty, otherwise False")    
     .def("reset", &hll_union::reset,

data/vendor/datasketches-cpp/python/src/kll_wrapper.cpp

@@ -130,6 +130,8 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "Produces a string summary of the sketch")
     .def("is_empty", &kll_sketch<T>::is_empty,
          "Returns True if the sketch is empty, otherwise False")
+    .def("get_k", &kll_sketch<T>::get_k,
+         "Returns the configured parameter k")
     .def("get_n", &kll_sketch<T>::get_n,
          "Returns the length of the input stream")
     .def("get_num_retained", &kll_sketch<T>::get_num_retained,
@@ -198,7 +200,7 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "If pmf is True, returns the 'double-sided' normalized rank error for the get_PMF() function.\n"
          "Otherwise, it is the 'single-sided' normalized rank error for all the other queries.\n"
          "Constants were derived as the best fit to 99 percentile empirically measured max error in thousands of trials")
-    .def("serialize", &dspy::kll_sketch_serialize<T>, "Serailizes the sketch into a bytes object")
+    .def("serialize", &dspy::kll_sketch_serialize<T>, "Serializes the sketch into a bytes object")
     .def_static("deserialize", &dspy::kll_sketch_deserialize<T>, "Deserializes the sketch from a bytes object")
     ;
 }