datasketches 0.2.4 → 0.2.5

data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp

@@ -22,6 +22,7 @@
 #include <cstring>
 #include <sstream>
 #include <fstream>
+#include <stdexcept>
 
 #include <kll_sketch.hpp>
 #include <test_allocator.hpp>
@@ -90,7 +91,9 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch.get_n() == 1);
     REQUIRE(sketch.get_num_retained() == 1);
     REQUIRE(sketch.get_rank(1.0f) == 0.0);
+    REQUIRE(sketch.get_rank<true>(1.0f) == 1.0);
     REQUIRE(sketch.get_rank(2.0f) == 1.0);
+    REQUIRE(sketch.get_rank(std::numeric_limits<float>::infinity()) == 1.0);
     REQUIRE(sketch.get_min_value() == 1.0);
     REQUIRE(sketch.get_max_value() == 1.0);
     REQUIRE(sketch.get_quantile(0.5) == 1.0);
@@ -142,8 +145,10 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(quantiles[2] == n - 1 );
 
     for (uint32_t i = 0; i < n; i++) {
-      const double trueRank = (double) i / n;
-      REQUIRE(sketch.get_rank(static_cast<float>(i)) == trueRank);
+      const double true_rank = (double) i / n;
+      REQUIRE(sketch.get_rank(static_cast<float>(i)) == true_rank);
+      const double true_rank_inclusive = (double) (i + 1) / n;
+      REQUIRE(sketch.get_rank<true>(static_cast<float>(i)) == true_rank_inclusive);
     }
 
     // the alternative method must produce the same result
@@ -241,20 +246,38 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
       sketch.update(static_cast<float>(i));
       values[i] = static_cast<float>(i);
     }
-
-    const auto ranks(sketch.get_CDF(values, n));
-    const auto pmf(sketch.get_PMF(values, n));
-
-    double subtotal_pmf(0);
-    for (int i = 0; i < n; i++) {
-      if (sketch.get_rank(values[i]) != ranks[i]) {
-        std::cerr << "checking rank vs CDF for value " << i << std::endl;
-        REQUIRE(sketch.get_rank(values[i]) == ranks[i]);
+    { // inclusive=false (default)
+      const auto ranks(sketch.get_CDF(values, n));
+      const auto pmf(sketch.get_PMF(values, n));
+
+      double subtotal_pmf = 0;
+      for (int i = 0; i < n; i++) {
+        if (sketch.get_rank(values[i]) != ranks[i]) {
+          std::cerr << "checking rank vs CDF for value " << i << std::endl;
+          REQUIRE(sketch.get_rank(values[i]) == ranks[i]);
+        }
+        subtotal_pmf += pmf[i];
+        if (abs(ranks[i] - subtotal_pmf) > NUMERIC_NOISE_TOLERANCE) {
+          std::cerr << "CDF vs PMF for value " << i << std::endl;
+          REQUIRE(ranks[i] == Approx(subtotal_pmf).margin(NUMERIC_NOISE_TOLERANCE));
+        }
       }
-      subtotal_pmf += pmf[i];
-      if (abs(ranks[i] - subtotal_pmf) > NUMERIC_NOISE_TOLERANCE) {
-        std::cerr << "CDF vs PMF for value " << i << std::endl;
-        REQUIRE(ranks[i] == Approx(subtotal_pmf).margin(NUMERIC_NOISE_TOLERANCE));
+    }
+    {  // inclusive=true
+      const auto ranks(sketch.get_CDF<true>(values, n));
+      const auto pmf(sketch.get_PMF<true>(values, n));
+
+      double subtotal_pmf = 0;
+      for (int i = 0; i < n; i++) {
+        if (sketch.get_rank<true>(values[i]) != ranks[i]) {
+          std::cerr << "checking rank vs CDF for value " << i << std::endl;
+          REQUIRE(sketch.get_rank(values[i]) == ranks[i]);
+        }
+        subtotal_pmf += pmf[i];
+        if (abs(ranks[i] - subtotal_pmf) > NUMERIC_NOISE_TOLERANCE) {
+          std::cerr << "CDF vs PMF for value " << i << std::endl;
+          REQUIRE(ranks[i] == Approx(subtotal_pmf).margin(NUMERIC_NOISE_TOLERANCE));
+        }
       }
     }
   }
@@ -293,7 +316,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   SECTION("bytes serialize deserialize empty") {
     kll_float_sketch sketch(200, 0);
     auto bytes = sketch.serialize();
-    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde<float>(), 0);
     REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
@@ -311,7 +334,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
+    auto sketch2 = kll_float_sketch::deserialize(s, serde<float>(), 0);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
     REQUIRE(s.tellg() == s.tellp());
     REQUIRE_FALSE(sketch2.is_empty());
@@ -330,7 +353,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     sketch.update(1.0f);
     auto bytes = sketch.serialize();
     REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde<float>(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
     REQUIRE_FALSE(sketch2.is_empty());
     REQUIRE_FALSE(sketch2.is_estimation_mode());
@@ -347,7 +370,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     std::ifstream is;
     is.exceptions(std::ios::failbit | std::ios::badbit);
     is.open(testBinaryInputPath + "kll_sketch_float_one_item_v1.sk", std::ios::binary);
-    auto sketch = kll_float_sketch::deserialize(is, test_allocator<float>(0));
+    auto sketch = kll_float_sketch::deserialize(is, serde<float>(), 0);
     REQUIRE_FALSE(sketch.is_empty());
     REQUIRE_FALSE(sketch.is_estimation_mode());
     REQUIRE(sketch.get_n() == 1);
@@ -364,7 +387,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
+    auto sketch2 = kll_float_sketch::deserialize(s, serde<float>(), 0);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
     REQUIRE(s.tellg() == s.tellp());
     REQUIRE_FALSE(sketch2.is_empty());
@@ -382,7 +405,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     sketch.update(3.0f);
     auto bytes = sketch.serialize();
     REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde<float>(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
     REQUIRE_FALSE(sketch2.is_empty());
     REQUIRE_FALSE(sketch2.is_estimation_mode());
@@ -399,7 +422,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
+    auto sketch2 = kll_float_sketch::deserialize(s, serde<float>(), 0);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
     REQUIRE(s.tellg() == s.tellp());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
@@ -421,7 +444,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     for (int i = 0; i < n; i++) sketch.update(static_cast<float>(i));
     auto bytes = sketch.serialize();
     REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde<float>(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
@@ -678,7 +701,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
     auto bytes = sketch1.serialize();
     REQUIRE(bytes.size() == sketch1.get_serialized_size_bytes());
-    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size(), serde<std::string>(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
     REQUIRE(sketch2.is_empty() == sketch1.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch1.is_estimation_mode());
@@ -699,7 +722,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     sketch1.update("a");
     auto bytes = sketch1.serialize();
     REQUIRE(bytes.size() == sketch1.get_serialized_size_bytes());
-    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size(), serde<std::string>(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
   }
 
@@ -766,6 +789,52 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     auto kll2 = kll_sketch<int8_t>::deserialize(blob.data(), blob.size());
   }
 
+  SECTION("sorted view") {
+    kll_sketch<int> kll;
+    kll.update(2);
+    kll.update(3);
+    kll.update(1);
+
+    { // non-cumulative, using operator->
+      auto view = kll.get_sorted_view(false);
+      REQUIRE(view.size() == 3);
+      auto it = view.begin();
+      REQUIRE(it->first == 1);
+      REQUIRE(it->second == 1);
+      ++it;
+      REQUIRE(it->first == 2);
+      REQUIRE(it->second == 1);
+      ++it;
+      REQUIRE(it->first == 3);
+      REQUIRE(it->second == 1);
+    }
+    { // cumulative, non-inclusive, using operator->
+      auto view = kll.get_sorted_view(true);
+      REQUIRE(view.size() == 3);
+      auto it = view.begin();
+      REQUIRE(it->first == 1);
+      REQUIRE(it->second == 0);
+      ++it;
+      REQUIRE(it->first == 2);
+      REQUIRE(it->second == 1);
+      ++it;
+      REQUIRE(it->first == 3);
+      REQUIRE(it->second == 2);
+    }
+    { // cumulative, inclusive, using operator*
+      auto view = kll.get_sorted_view<true>(true);
+      REQUIRE(view.size() == 3);
+      auto it = view.begin();
+      REQUIRE((*it).first == 1);
+      REQUIRE((*it).second == 1);
+      ++it;
+      REQUIRE((*it).first == 2);
+      REQUIRE((*it).second == 2);
+      ++it;
+      REQUIRE((*it).first == 3);
+      REQUIRE((*it).second == 3);
+    }
+  }
   // cleanup
   if (test_allocator_total_bytes != 0) {
     REQUIRE(test_allocator_total_bytes == 0);

data/vendor/datasketches-cpp/kll/test/kolmogorov_smirnov_test.cpp

@@ -46,7 +46,7 @@ TEST_CASE("kolmogorov-smirnov same distribution", "[kll_sketch]") {
     sketch1.update(x);
     sketch2.update(x);
   }
-  REQUIRE(kolmogorov_smirnov::delta(sketch1, sketch2) == Approx(0).margin(0.01));
+  REQUIRE(kolmogorov_smirnov::delta(sketch1, sketch2) == Approx(0).margin(0.02));
   REQUIRE_FALSE(kolmogorov_smirnov::test(sketch1, sketch2, 0.01));
 }
 

data/vendor/datasketches-cpp/pyproject.toml

@@ -1,7 +1,7 @@
 [build-system]
 requires = ["wheel",
             "setuptools >= 30.3.0",
-            "cmake >= 3.12",
+            "cmake >= 3.16",
             "pip >= 10.0",
             "pybind11[global] >= 2.6.0"]
 build-backend = "setuptools.build_meta"

data/vendor/datasketches-cpp/python/CMakeLists.txt

@@ -15,7 +15,11 @@
 # specific language governing permissions and limitations
 # under the License.
 
-find_package(Python3 COMPONENTS Interpreter Development)
+if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18.0")
+find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED)
+else()
+find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
+endif()
 
 # only Windows+MSVC seems to have trouble locating pybind11
 if (MSVC)
@@ -40,6 +44,7 @@ target_link_libraries(python
     theta
     sampling
     req
+    quantiles
     pybind11::module
 )
 
@@ -63,5 +68,7 @@ target_sources(python
     src/theta_wrapper.cpp
     src/vo_wrapper.cpp
     src/req_wrapper.cpp
+    src/quantiles_wrapper.cpp
+    src/ks_wrapper.cpp
     src/vector_of_kll.cpp
 )

data/vendor/datasketches-cpp/python/README.md

@@ -27,6 +27,11 @@ Having installed the library, loading the Apache Datasketches Library in Python
 - KLL (Absolute Error Quantiles)
   - `kll_ints_sketch`
   - `kll_floats_sketch`
+  - `kll_doubles_sketch`
+- Quantiles (Absolute Error Quantiles, inferior algorithm)
+  - `quantiles_ints_sketch`
+  - `quantiles_floats_sketch`
+  - `quantiles_doubles_sketch`
 - REQ (Relative Error Quantiles)
   - `req_ints_sketch`
   - `req_floats_sketch`
@@ -52,6 +57,8 @@ Having installed the library, loading the Apache Datasketches Library in Python
 - Vector of KLL
   - `vector_of_kll_ints_sketches`
   - `vector_of_kll_floats_sketches`
+- Kolmogorov-Smirnov Test
+  - `ks_test` applied to a pair of matched-type Absolute Error quantiles sketches
 
 ## Known Differences from C++
 

data/vendor/datasketches-cpp/python/src/datasketches.cpp

@@ -28,6 +28,8 @@ void init_cpc(py::module& m);
 void init_theta(py::module& m);
 void init_vo(py::module& m);
 void init_req(py::module& m);
+void init_quantiles(py::module& m);
+void init_kolmogorov_smirnov(py::module& m);
 void init_vector_of_kll(py::module& m);
 
 PYBIND11_MODULE(datasketches, m) {
@@ -38,5 +40,7 @@ PYBIND11_MODULE(datasketches, m) {
   init_theta(m);
   init_vo(m);
   init_req(m);
+  init_quantiles(m);
+  init_kolmogorov_smirnov(m);
   init_vector_of_kll(m);
 }

data/vendor/datasketches-cpp/python/src/fi_wrapper.cpp

@@ -64,6 +64,11 @@ py::list fi_sketch_get_frequent_items(const frequent_items_sketch<T>& sk,
   return list;
 }
 
+template<typename T>
+size_t fi_sketch_get_serialized_size_bytes(const frequent_items_sketch<T>& sk) {
+  return sk.get_serialized_size_bytes();
+}
+
 }
 }
 
@@ -104,7 +109,7 @@ void bind_fi_sketch(py::module &m, const char* name) {
          "Returns the epsilon value used to compute a priori error for a given log2(max_map_size)")
     .def_static("get_apriori_error", &frequent_items_sketch<T>::get_apriori_error, py::arg("lg_max_map_size"), py::arg("estimated_total_weight"),
          "Returns the estimated a priori error given the max_map_size for the sketch and the estimated_total_stream_weight.")
-    .def("get_serialized_size_bytes", &frequent_items_sketch<T>::get_serialized_size_bytes,
+    .def("get_serialized_size_bytes", &dspy::fi_sketch_get_serialized_size_bytes<T>,
          "Computes the size needed to serialize the current state of the sketch. This can be expensive since every item needs to be looked at.")
     .def("serialize", &dspy::fi_sketch_serialize<T>, "Serializes the sketch into a bytes object")
     .def_static("deserialize", &dspy::fi_sketch_deserialize<T>, "Reads a bytes object and returns the corresponding frequent_strings_sketch")

data/vendor/datasketches-cpp/python/src/kll_wrapper.cpp

@@ -24,6 +24,7 @@
 #include <pybind11/numpy.h>
 #include <sstream>
 #include <vector>
+#include <stdexcept>
 
 namespace py = pybind11;
 
@@ -50,11 +51,32 @@ double kll_sketch_generic_normalized_rank_error(uint16_t k, bool pmf) {
   return kll_sketch<T>::get_normalized_rank_error(k, pmf);
 }
 
+template<typename T>
+double kll_sketch_get_rank(const kll_sketch<T>& sk, const T& item, bool inclusive) {
+  if (inclusive)
+    return sk.template get_rank<true>(item);
+  else
+    return sk.template get_rank<false>(item);
+}
+
+template<typename T>
+T kll_sketch_get_quantile(const kll_sketch<T>& sk,
+                          double rank,
+                          bool inclusive) {
+  if (inclusive)
+    return T(sk.template get_quantile<true>(rank));
+  else
+    return T(sk.template get_quantile<false>(rank));
+}
+
 template<typename T>
 py::list kll_sketch_get_quantiles(const kll_sketch<T>& sk,
-                                  std::vector<double>& fractions) {
+                                  std::vector<double>& fractions,
+                                  bool inclusive) {
   size_t nQuantiles = fractions.size();
-  auto result = sk.get_quantiles(&fractions[0], nQuantiles);
+  auto result = inclusive ?
+      sk.template get_quantiles<true>(fractions.data(), nQuantiles)
+    : sk.template get_quantiles<false>(fractions.data(), nQuantiles);
 
   // returning as std::vector<> would copy values to a list anyway
   py::list list(nQuantiles);
@@ -67,9 +89,12 @@ py::list kll_sketch_get_quantiles(const kll_sketch<T>& sk,
 
 template<typename T>
 py::list kll_sketch_get_pmf(const kll_sketch<T>& sk,
-                            std::vector<T>& split_points) {
+                            std::vector<T>& split_points,
+                            bool inclusive) {
   size_t nPoints = split_points.size();
-  auto result = sk.get_PMF(&split_points[0], nPoints);
+  auto result = inclusive ?
+      sk.template get_PMF<true>(split_points.data(), nPoints)
+    : sk.template get_PMF<false>(split_points.data(), nPoints);
 
   py::list list(nPoints + 1);
   for (size_t i = 0; i <= nPoints; ++i) {
@@ -81,9 +106,12 @@ py::list kll_sketch_get_pmf(const kll_sketch<T>& sk,
 
 template<typename T>
 py::list kll_sketch_get_cdf(const kll_sketch<T>& sk,
-                            std::vector<T>& split_points) {
+                            std::vector<T>& split_points,
+                            bool inclusive) {
   size_t nPoints = split_points.size();
-  auto result = sk.get_CDF(&split_points[0], nPoints);
+  auto result = inclusive ?
+      sk.template get_CDF<true>(split_points.data(), nPoints)
+    : sk.template get_CDF<false>(split_points.data(), nPoints);
 
   py::list list(nPoints + 1);
   for (size_t i = 0; i <= nPoints; ++i) {
@@ -142,7 +170,7 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "Returns the minimum value from the stream. If empty, kll_floats_sketch retursn nan; kll_ints_sketch throws a RuntimeError")
     .def("get_max_value", &kll_sketch<T>::get_max_value,
          "Returns the maximum value from the stream. If empty, kll_floats_sketch retursn nan; kll_ints_sketch throws a RuntimeError")
-    .def("get_quantile", &kll_sketch<T>::get_quantile, py::arg("fraction"),
+    .def("get_quantile", &dspy::kll_sketch_get_quantile<T>, py::arg("fraction"), py::arg("inclusive")=false,
          "Returns an approximation to the value of the data item "
          "that would be preceded by the given fraction of a hypothetical sorted "
          "version of the input stream so far.\n"
@@ -151,7 +179,7 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "sketch. Instead use get_quantiles(), which pays the overhead only once.\n"
          "For kll_floats_sketch: if the sketch is empty this returns nan. "
          "For kll_ints_sketch: if the sketch is empty this throws a RuntimeError.")
-    .def("get_quantiles", &dspy::kll_sketch_get_quantiles<T>, py::arg("fractions"),
+    .def("get_quantiles", &dspy::kll_sketch_get_quantiles<T>, py::arg("fractions"), py::arg("inclusive")=false,
          "This is a more efficient multiple-query version of get_quantile().\n"
          "This returns an array that could have been generated by using get_quantile() for each "
          "fractional rank separately, but would be very inefficient. "
@@ -159,12 +187,14 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "a single query. It is strongly recommend that this method be used instead of multiple calls "
          "to get_quantile().\n"
          "If the sketch is empty this returns an empty vector.")
-    .def("get_rank", &kll_sketch<T>::get_rank, py::arg("value"),
+    .def("get_rank", &dspy::kll_sketch_get_rank<T>, py::arg("value"), py::arg("inclusive")=false,
          "Returns an approximation to the normalized (fractional) rank of the given value from 0 to 1, inclusive.\n"
          "The resulting approximation has a probabilistic guarantee that can be obtained from the "
          "get_normalized_rank_error(False) function.\n"
+         "With the parameter inclusive=true the weight of the given value is included into the rank."
+         "Otherwise the rank equals the sum of the weights of values less than the given value.\n"
          "If the sketch is empty this returns nan.")
-    .def("get_pmf", &dspy::kll_sketch_get_pmf<T>, py::arg("split_points"),
+    .def("get_pmf", &dspy::kll_sketch_get_pmf<T>, py::arg("split_points"), py::arg("inclusive")=false,
          "Returns an approximation to the Probability Mass Function (PMF) of the input stream "
          "given a set of split points (values).\n"
          "The resulting approximations have a probabilistic guarantee that can be obtained from the "
@@ -172,11 +202,13 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "If the sketch is empty this returns an empty vector.\n"
          "split_points is an array of m unique, monotonically increasing float values "
          "that divide the real number line into m+1 consecutive disjoint intervals.\n"
-         "The definition of an 'interval' is inclusive of the left split point (or minimum value) and "
+         "If the parameter inclusive=false, the definition of an 'interval' is inclusive of the left split point (or minimum value) and "
          "exclusive of the right split point, with the exception that the last interval will include "
          "the maximum value.\n"
+         "If the parameter inclusive=true, the definition of an 'interval' is exclusive of the left split point (or minimum value) and "
+         "inclusive of the right split point.\n"
          "It is not necessary to include either the min or max values in these split points.")
-    .def("get_cdf", &dspy::kll_sketch_get_cdf<T>, py::arg("split_points"),
+    .def("get_cdf", &dspy::kll_sketch_get_cdf<T>, py::arg("split_points"), py::arg("inclusive")=false,
          "Returns an approximation to the Cumulative Distribution Function (CDF), which is the "
          "cumulative analog of the PMF, of the input stream given a set of split points (values).\n"
          "The resulting approximations have a probabilistic guarantee that can be obtained from the "
@@ -184,9 +216,11 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "If the sketch is empty this returns an empty vector.\n"
          "split_points is an array of m unique, monotonically increasing float values "
          "that divide the real number line into m+1 consecutive disjoint intervals.\n"
-         "The definition of an 'interval' is inclusive of the left split point (or minimum value) and "
+         "If the parameter inclusive=false, the definition of an 'interval' is inclusive of the left split point (or minimum value) and "
          "exclusive of the right split point, with the exception that the last interval will include "
          "the maximum value.\n"
+         "If the parameter inclusive=true, the definition of an 'interval' is exclusive of the left split point (or minimum value) and "
+         "inclusive of the right split point.\n"
          "It is not necessary to include either the min or max values in these split points.")
     .def("normalized_rank_error", (double (kll_sketch<T>::*)(bool) const) &kll_sketch<T>::get_normalized_rank_error,
          py::arg("as_pmf"),
@@ -208,4 +242,5 @@ void bind_kll_sketch(py::module &m, const char* name) {
 void init_kll(py::module &m) {
   bind_kll_sketch<int>(m, "kll_ints_sketch");
   bind_kll_sketch<float>(m, "kll_floats_sketch");
+  bind_kll_sketch<double>(m, "kll_doubles_sketch");
 }

data/vendor/datasketches-cpp/python/src/ks_wrapper.cpp

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "kolmogorov_smirnov.hpp"
+#include "kll_sketch.hpp"
+#include "quantiles_sketch.hpp"
+
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+
+void init_kolmogorov_smirnov(py::module &m) {
+  using namespace datasketches;
+
+  m.def("ks_test", &kolmogorov_smirnov::test<kll_sketch<int>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between kll_ints_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+  m.def("ks_test", &kolmogorov_smirnov::test<kll_sketch<float>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between kll_floats_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+  m.def("ks_test", &kolmogorov_smirnov::test<kll_sketch<double>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between kll_doubles_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+
+  m.def("ks_test", &kolmogorov_smirnov::test<quantiles_sketch<int>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between quantiles_ints_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+  m.def("ks_test", &kolmogorov_smirnov::test<quantiles_sketch<float>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between quantiles_floats_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+  m.def("ks_test", &kolmogorov_smirnov::test<quantiles_sketch<double>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between quantiles_doubles_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+}