datasketches 0.2.2 → 0.2.5

data/vendor/datasketches-cpp/python/src/kll_wrapper.cpp

@@ -24,6 +24,7 @@
 #include <pybind11/numpy.h>
 #include <sstream>
 #include <vector>
+#include <stdexcept>
 
 namespace py = pybind11;
 
@@ -50,11 +51,32 @@ double kll_sketch_generic_normalized_rank_error(uint16_t k, bool pmf) {
   return kll_sketch<T>::get_normalized_rank_error(k, pmf);
 }
 
+template<typename T>
+double kll_sketch_get_rank(const kll_sketch<T>& sk, const T& item, bool inclusive) {
+  if (inclusive)
+    return sk.template get_rank<true>(item);
+  else
+    return sk.template get_rank<false>(item);
+}
+
+template<typename T>
+T kll_sketch_get_quantile(const kll_sketch<T>& sk,
+                          double rank,
+                          bool inclusive) {
+  if (inclusive)
+    return T(sk.template get_quantile<true>(rank));
+  else
+    return T(sk.template get_quantile<false>(rank));
+}
+
 template<typename T>
 py::list kll_sketch_get_quantiles(const kll_sketch<T>& sk,
-                                  std::vector<double>& fractions) {
+                                  std::vector<double>& fractions,
+                                  bool inclusive) {
   size_t nQuantiles = fractions.size();
-  auto result = sk.get_quantiles(&fractions[0], nQuantiles);
+  auto result = inclusive ?
+      sk.template get_quantiles<true>(fractions.data(), nQuantiles)
+    : sk.template get_quantiles<false>(fractions.data(), nQuantiles);
 
   // returning as std::vector<> would copy values to a list anyway
   py::list list(nQuantiles);
@@ -67,9 +89,12 @@ py::list kll_sketch_get_quantiles(const kll_sketch<T>& sk,
 
 template<typename T>
 py::list kll_sketch_get_pmf(const kll_sketch<T>& sk,
-                            std::vector<T>& split_points) {
+                            std::vector<T>& split_points,
+                            bool inclusive) {
   size_t nPoints = split_points.size();
-  auto result = sk.get_PMF(&split_points[0], nPoints);
+  auto result = inclusive ?
+      sk.template get_PMF<true>(split_points.data(), nPoints)
+    : sk.template get_PMF<false>(split_points.data(), nPoints);
 
   py::list list(nPoints + 1);
   for (size_t i = 0; i <= nPoints; ++i) {
@@ -81,9 +106,12 @@ py::list kll_sketch_get_pmf(const kll_sketch<T>& sk,
 
 template<typename T>
 py::list kll_sketch_get_cdf(const kll_sketch<T>& sk,
-                            std::vector<T>& split_points) {
+                            std::vector<T>& split_points,
+                            bool inclusive) {
   size_t nPoints = split_points.size();
-  auto result = sk.get_CDF(&split_points[0], nPoints);
+  auto result = inclusive ?
+      sk.template get_CDF<true>(split_points.data(), nPoints)
+    : sk.template get_CDF<false>(split_points.data(), nPoints);
 
   py::list list(nPoints + 1);
   for (size_t i = 0; i <= nPoints; ++i) {
@@ -116,7 +144,7 @@ void bind_kll_sketch(py::module &m, const char* name) {
   using namespace datasketches;
 
   py::class_<kll_sketch<T>>(m, name)
-    .def(py::init<uint16_t>(), py::arg("k")=kll_sketch<T>::DEFAULT_K)
+    .def(py::init<uint16_t>(), py::arg("k")=kll_constants::DEFAULT_K)
     .def(py::init<const kll_sketch<T>&>())
     .def("update", (void (kll_sketch<T>::*)(const T&)) &kll_sketch<T>::update, py::arg("item"),
          "Updates the sketch with the given value")
@@ -142,7 +170,7 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "Returns the minimum value from the stream. If empty, kll_floats_sketch retursn nan; kll_ints_sketch throws a RuntimeError")
     .def("get_max_value", &kll_sketch<T>::get_max_value,
          "Returns the maximum value from the stream. If empty, kll_floats_sketch retursn nan; kll_ints_sketch throws a RuntimeError")
-    .def("get_quantile", &kll_sketch<T>::get_quantile, py::arg("fraction"),
+    .def("get_quantile", &dspy::kll_sketch_get_quantile<T>, py::arg("fraction"), py::arg("inclusive")=false,
          "Returns an approximation to the value of the data item "
          "that would be preceded by the given fraction of a hypothetical sorted "
          "version of the input stream so far.\n"
@@ -151,7 +179,7 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "sketch. Instead use get_quantiles(), which pays the overhead only once.\n"
          "For kll_floats_sketch: if the sketch is empty this returns nan. "
          "For kll_ints_sketch: if the sketch is empty this throws a RuntimeError.")
-    .def("get_quantiles", &dspy::kll_sketch_get_quantiles<T>, py::arg("fractions"),
+    .def("get_quantiles", &dspy::kll_sketch_get_quantiles<T>, py::arg("fractions"), py::arg("inclusive")=false,
          "This is a more efficient multiple-query version of get_quantile().\n"
          "This returns an array that could have been generated by using get_quantile() for each "
          "fractional rank separately, but would be very inefficient. "
@@ -159,12 +187,14 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "a single query. It is strongly recommend that this method be used instead of multiple calls "
          "to get_quantile().\n"
          "If the sketch is empty this returns an empty vector.")
-    .def("get_rank", &kll_sketch<T>::get_rank, py::arg("value"),
+    .def("get_rank", &dspy::kll_sketch_get_rank<T>, py::arg("value"), py::arg("inclusive")=false,
          "Returns an approximation to the normalized (fractional) rank of the given value from 0 to 1, inclusive.\n"
          "The resulting approximation has a probabilistic guarantee that can be obtained from the "
          "get_normalized_rank_error(False) function.\n"
+         "With the parameter inclusive=true the weight of the given value is included into the rank."
+         "Otherwise the rank equals the sum of the weights of values less than the given value.\n"
          "If the sketch is empty this returns nan.")
-    .def("get_pmf", &dspy::kll_sketch_get_pmf<T>, py::arg("split_points"),
+    .def("get_pmf", &dspy::kll_sketch_get_pmf<T>, py::arg("split_points"), py::arg("inclusive")=false,
          "Returns an approximation to the Probability Mass Function (PMF) of the input stream "
          "given a set of split points (values).\n"
          "The resulting approximations have a probabilistic guarantee that can be obtained from the "
@@ -172,11 +202,13 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "If the sketch is empty this returns an empty vector.\n"
          "split_points is an array of m unique, monotonically increasing float values "
          "that divide the real number line into m+1 consecutive disjoint intervals.\n"
-         "The definition of an 'interval' is inclusive of the left split point (or minimum value) and "
+         "If the parameter inclusive=false, the definition of an 'interval' is inclusive of the left split point (or minimum value) and "
          "exclusive of the right split point, with the exception that the last interval will include "
          "the maximum value.\n"
+         "If the parameter inclusive=true, the definition of an 'interval' is exclusive of the left split point (or minimum value) and "
+         "inclusive of the right split point.\n"
          "It is not necessary to include either the min or max values in these split points.")
-    .def("get_cdf", &dspy::kll_sketch_get_cdf<T>, py::arg("split_points"),
+    .def("get_cdf", &dspy::kll_sketch_get_cdf<T>, py::arg("split_points"), py::arg("inclusive")=false,
          "Returns an approximation to the Cumulative Distribution Function (CDF), which is the "
          "cumulative analog of the PMF, of the input stream given a set of split points (values).\n"
          "The resulting approximations have a probabilistic guarantee that can be obtained from the "
@@ -184,9 +216,11 @@ void bind_kll_sketch(py::module &m, const char* name) {
          "If the sketch is empty this returns an empty vector.\n"
          "split_points is an array of m unique, monotonically increasing float values "
          "that divide the real number line into m+1 consecutive disjoint intervals.\n"
-         "The definition of an 'interval' is inclusive of the left split point (or minimum value) and "
+         "If the parameter inclusive=false, the definition of an 'interval' is inclusive of the left split point (or minimum value) and "
          "exclusive of the right split point, with the exception that the last interval will include "
          "the maximum value.\n"
+         "If the parameter inclusive=true, the definition of an 'interval' is exclusive of the left split point (or minimum value) and "
+         "inclusive of the right split point.\n"
          "It is not necessary to include either the min or max values in these split points.")
     .def("normalized_rank_error", (double (kll_sketch<T>::*)(bool) const) &kll_sketch<T>::get_normalized_rank_error,
          py::arg("as_pmf"),
@@ -208,4 +242,5 @@ void bind_kll_sketch(py::module &m, const char* name) {
 void init_kll(py::module &m) {
   bind_kll_sketch<int>(m, "kll_ints_sketch");
   bind_kll_sketch<float>(m, "kll_floats_sketch");
+  bind_kll_sketch<double>(m, "kll_doubles_sketch");
 }

data/vendor/datasketches-cpp/python/src/ks_wrapper.cpp

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "kolmogorov_smirnov.hpp"
+#include "kll_sketch.hpp"
+#include "quantiles_sketch.hpp"
+
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+
+void init_kolmogorov_smirnov(py::module &m) {
+  using namespace datasketches;
+
+  m.def("ks_test", &kolmogorov_smirnov::test<kll_sketch<int>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between kll_ints_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+  m.def("ks_test", &kolmogorov_smirnov::test<kll_sketch<float>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between kll_floats_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+  m.def("ks_test", &kolmogorov_smirnov::test<kll_sketch<double>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between kll_doubles_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+
+  m.def("ks_test", &kolmogorov_smirnov::test<quantiles_sketch<int>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between quantiles_ints_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+  m.def("ks_test", &kolmogorov_smirnov::test<quantiles_sketch<float>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between quantiles_floats_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+  m.def("ks_test", &kolmogorov_smirnov::test<quantiles_sketch<double>>, py::arg("sk_1"), py::arg("sk_2"), py::arg("p"),
+    "Performs the Kolmogorov-Smirnov Test between quantiles_doubles_sketches.\n"
+    "Note: if the given sketches have insufficient data or if the sketch sizes are too small, "
+    "this will return false.\n"
+    "Returns True if we can reject the null hypothesis (that the sketches reflect the same underlying "
+    "distribution) using the provided p-value, otherwise False.");
+}

data/vendor/datasketches-cpp/python/src/quantiles_wrapper.cpp

@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "quantiles_sketch.hpp"
+
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/numpy.h>
+#include <vector>
+
+namespace py = pybind11;
+
+namespace datasketches {
+
+namespace python {
+
+template<typename T>
+quantiles_sketch<T> quantiles_sketch_deserialize(py::bytes sk_bytes) {
+  std::string sk_str = sk_bytes; // implicit cast  
+  return quantiles_sketch<T>::deserialize(sk_str.c_str(), sk_str.length());
+}
+
+template<typename T>
+py::object quantiles_sketch_serialize(const quantiles_sketch<T>& sk) {
+  auto ser_result = sk.serialize();
+  return py::bytes((char*)ser_result.data(), ser_result.size());
+}
+
+// maybe possible to disambiguate the static vs method rank error calls, but
+// this is easier for now
+template<typename T>
+double quantiles_sketch_generic_normalized_rank_error(uint16_t k, bool pmf) {
+  return quantiles_sketch<T>::get_normalized_rank_error(k, pmf);
+}
+
+template<typename T>
+double quantiles_sketch_get_rank(const quantiles_sketch<T>& sk,
+                                 const T& item,
+                                 bool inclusive) {
+  if (inclusive)
+    return sk.template get_rank<true>(item);
+  else
+    return sk.template get_rank<false>(item);
+}
+
+template<typename T>
+T quantiles_sketch_get_quantile(const quantiles_sketch<T>& sk,
+                                double rank,
+                                bool inclusive) {
+  if (inclusive)
+    return T(sk.template get_quantile<true>(rank));
+  else
+    return T(sk.template get_quantile<false>(rank));
+}
+
+template<typename T>
+py::list quantiles_sketch_get_quantiles(const quantiles_sketch<T>& sk,
+                                        std::vector<double>& fractions,
+                                        bool inclusive) {
+  size_t n_quantiles = fractions.size();
+  auto result = inclusive
+     ? sk.template get_quantiles<true>(&fractions[0], static_cast<uint32_t>(n_quantiles))
+     : sk.template get_quantiles<false>(&fractions[0], static_cast<uint32_t>(n_quantiles));
+
+  // returning as std::vector<> would copy values to a list anyway
+  py::list list(n_quantiles);
+  for (size_t i = 0; i < n_quantiles; ++i) {
+      list[i] = result[i];
+  }
+
+  return list;
+}
+
+template<typename T>
+py::list quantiles_sketch_get_pmf(const quantiles_sketch<T>& sk,
+                                  std::vector<T>& split_points,
+                                  bool inclusive) {
+  size_t n_points = split_points.size();
+  auto result = inclusive
+     ? sk.template get_PMF<true>(&split_points[0], n_points)
+     : sk.template get_PMF<false>(&split_points[0], n_points);
+
+  py::list list(n_points + 1);
+  for (size_t i = 0; i <= n_points; ++i) {
+    list[i] = result[i];
+  }
+
+  return list;
+}
+
+template<typename T>
+py::list quantiles_sketch_get_cdf(const quantiles_sketch<T>& sk,
+                                  std::vector<T>& split_points,
+                                  bool inclusive) {
+  size_t n_points = split_points.size();
+  auto result = inclusive
+     ? sk.template get_CDF<true>(&split_points[0], n_points)
+     : sk.template get_CDF<false>(&split_points[0], n_points);
+
+  py::list list(n_points + 1);
+  for (size_t i = 0; i <= n_points; ++i) {
+    list[i] = result[i];
+  }
+
+  return list;
+}
+
+template<typename T>
+void quantiles_sketch_update(quantiles_sketch<T>& sk, py::array_t<T, py::array::c_style | py::array::forcecast> items) {
+  if (items.ndim() != 1) {
+    throw std::invalid_argument("input data must have only one dimension. Found: "
+          + std::to_string(items.ndim()));
+  }
+  
+  auto data = items.template unchecked<1>();
+  for (uint32_t i = 0; i < data.size(); ++i) {
+    sk.update(data(i));
+  }
+}
+
+}
+}
+
+namespace dspy = datasketches::python;
+
+template<typename T>
+void bind_quantiles_sketch(py::module &m, const char* name) {
+  using namespace datasketches;
+
+  py::class_<quantiles_sketch<T>>(m, name)
+    .def(py::init<uint16_t>(), py::arg("k")=quantiles_constants::DEFAULT_K)
+    .def(py::init<const quantiles_sketch<T>&>())
+    .def("update", (void (quantiles_sketch<T>::*)(const T&)) &quantiles_sketch<T>::update, py::arg("item"),
+         "Updates the sketch with the given value")
+    .def("update", &dspy::quantiles_sketch_update<T>, py::arg("array"),
+         "Updates the sketch with the values in the given array")
+    .def("merge", (void (quantiles_sketch<T>::*)(const quantiles_sketch<T>&)) &quantiles_sketch<T>::merge, py::arg("sketch"),
+         "Merges the provided sketch into the this one")
+    .def("__str__", &quantiles_sketch<T>::to_string, py::arg("print_levels")=false, py::arg("print_items")=false,
+         "Produces a string summary of the sketch")
+    .def("to_string", &quantiles_sketch<T>::to_string, py::arg("print_levels")=false, py::arg("print_items")=false,
+         "Produces a string summary of the sketch")
+    .def("is_empty", &quantiles_sketch<T>::is_empty,
+         "Returns True if the sketch is empty, otherwise False")
+    .def("get_k", &quantiles_sketch<T>::get_k,
+         "Returns the configured parameter k")
+    .def("get_n", &quantiles_sketch<T>::get_n,
+         "Returns the length of the input stream")
+    .def("get_num_retained", &quantiles_sketch<T>::get_num_retained,
+         "Returns the number of retained items (samples) in the sketch")
+    .def("is_estimation_mode", &quantiles_sketch<T>::is_estimation_mode,
+         "Returns True if the sketch is in estimation mode, otherwise False")
+    .def("get_min_value", &quantiles_sketch<T>::get_min_value,
+         "Returns the minimum value from the stream. If empty, quantiles_floats_sketch returns nan; quantiles_ints_sketch throws a RuntimeError")
+    .def("get_max_value", &quantiles_sketch<T>::get_max_value,
+         "Returns the maximum value from the stream. If empty, quantiles_floats_sketch returns nan; quantiles_ints_sketch throws a RuntimeError")
+    .def("get_quantile", &dspy::quantiles_sketch_get_quantile<T>, py::arg("rank"), py::arg("inclusive")=false,
+         "Returns an approximation to the value of the data item "
+         "that would be preceded by the given fraction of a hypothetical sorted "
+         "version of the input stream so far.\n"
+         "Note that this method has a fairly large overhead (microseconds instead of nanoseconds) "
+         "so it should not be called multiple times to get different quantiles from the same "
+         "sketch. Instead use get_quantiles(), which pays the overhead only once.\n"
+         "For quantiles_floats_sketch: if the sketch is empty this returns nan. "
+         "For quantiles_ints_sketch: if the sketch is empty this throws a RuntimeError.")
+    .def("get_quantiles", &dspy::quantiles_sketch_get_quantiles<T>, py::arg("ranks"), py::arg("inclusive")=false,
+         "This is a more efficient multiple-query version of get_quantile().\n"
+         "This returns an array that could have been generated by using get_quantile() for each "
+         "fractional rank separately, but would be very inefficient. "
+         "This method incurs the internal set-up overhead once and obtains multiple quantile values in "
+         "a single query. It is strongly recommend that this method be used instead of multiple calls "
+         "to get_quantile().\n"
+         "If the sketch is empty this returns an empty vector.")
+    .def("get_rank", &dspy::quantiles_sketch_get_rank<T>, py::arg("item"), py::arg("inclusive")=false,
+         "Returns an approximation to the normalized (fractional) rank of the given value from 0 to 1, inclusive.\n"
+         "The resulting approximation has a probabilistic guarantee that can be obtained from the "
+         "get_normalized_rank_error(False) function.\n"
+         "If the sketch is empty this returns nan.")
+    .def("get_pmf", &dspy::quantiles_sketch_get_pmf<T>, py::arg("split_points"), py::arg("inclusive")=false,
+         "Returns an approximation to the Probability Mass Function (PMF) of the input stream "
+         "given a set of split points (values).\n"
+         "The resulting approximations have a probabilistic guarantee that can be obtained from the "
+         "get_normalized_rank_error(True) function.\n"
+         "If the sketch is empty this returns an empty vector.\n"
+         "split_points is an array of m unique, monotonically increasing float values "
+         "that divide the real number line into m+1 consecutive disjoint intervals.\n"
+         "The definition of an 'interval' is inclusive of the left split point (or minimum value) and "
+         "exclusive of the right split point, with the exception that the last interval will include "
+         "the maximum value.\n"
+         "It is not necessary to include either the min or max values in these split points.")
+    .def("get_cdf", &dspy::quantiles_sketch_get_cdf<T>, py::arg("split_points"), py::arg("inclusive")=false,
+         "Returns an approximation to the Cumulative Distribution Function (CDF), which is the "
+         "cumulative analog of the PMF, of the input stream given a set of split points (values).\n"
+         "The resulting approximations have a probabilistic guarantee that can be obtained from the "
+         "get_normalized_rank_error(True) function.\n"
+         "If the sketch is empty this returns an empty vector.\n"
+         "split_points is an array of m unique, monotonically increasing float values "
+         "that divide the real number line into m+1 consecutive disjoint intervals.\n"
+         "The definition of an 'interval' is inclusive of the left split point (or minimum value) and "
+         "exclusive of the right split point, with the exception that the last interval will include "
+         "the maximum value.\n"
+         "It is not necessary to include either the min or max values in these split points.")
+    .def("normalized_rank_error", (double (quantiles_sketch<T>::*)(bool) const) &quantiles_sketch<T>::get_normalized_rank_error,
+         py::arg("as_pmf"),
+         "Gets the normalized rank error for this sketch.\n"
+         "If pmf is True, returns the 'double-sided' normalized rank error for the get_PMF() function.\n"
+         "Otherwise, it is the 'single-sided' normalized rank error for all the other queries.\n"
+         "Constants were derived as the best fit to 99 percentile empirically measured max error in thousands of trials")
+    .def_static("get_normalized_rank_error", &dspy::quantiles_sketch_generic_normalized_rank_error<T>,
+         py::arg("k"), py::arg("as_pmf"),
+         "Gets the normalized rank error given parameters k and the pmf flag.\n"
+         "If pmf is True, returns the 'double-sided' normalized rank error for the get_PMF() function.\n"
+         "Otherwise, it is the 'single-sided' normalized rank error for all the other queries.\n"
+         "Constants were derived as the best fit to 99 percentile empirically measured max error in thousands of trials")
+    .def("serialize", &dspy::quantiles_sketch_serialize<T>, "Serializes the sketch into a bytes object")
+    .def_static("deserialize", &dspy::quantiles_sketch_deserialize<T>, "Deserializes the sketch from a bytes object")
+    ;
+}
+
+void init_quantiles(py::module &m) {
+  bind_quantiles_sketch<int>(m, "quantiles_ints_sketch");
+  bind_quantiles_sketch<float>(m, "quantiles_floats_sketch");
+  bind_quantiles_sketch<double>(m, "quantiles_doubles_sketch");
+}

data/vendor/datasketches-cpp/python/src/req_wrapper.cpp

@@ -24,6 +24,7 @@
 #include <pybind11/numpy.h>
 #include <sstream>
 #include <vector>
+#include <stdexcept>
 
 namespace py = pybind11;
 
@@ -194,6 +195,8 @@ void bind_req_sketch(py::module &m, const char* name) {
          "Returns an approximation to the normalized (fractional) rank of the given value from 0 to 1, inclusive.\n"
          "The resulting approximation has a probabilistic guarantee that can be obtained from the "
          "get_normalized_rank_error(False) function.\n"
+         "With the parameter inclusive=true the weight of the given item is included into the rank."
+         "Otherwise the rank equals the sum of the weights of items less than the given item.\n"
          "If the sketch is empty this returns nan.")
     .def("get_pmf", &dspy::req_sketch_get_pmf<T>, py::arg("split_points"), py::arg("inclusive")=false,
          "Returns an approximation to the Probability Mass Function (PMF) of the input stream "
@@ -203,9 +206,11 @@ void bind_req_sketch(py::module &m, const char* name) {
          "If the sketch is empty this returns an empty vector.\n"
          "split_points is an array of m unique, monotonically increasing float values "
          "that divide the real number line into m+1 consecutive disjoint intervals.\n"
-         "The definition of an 'interval' is inclusive of the left split point (or minimum value) and "
+         "If the parameter inclusive=false, the definition of an 'interval' is inclusive of the left split point (or minimum value) and "
          "exclusive of the right split point, with the exception that the last interval will include "
          "the maximum value.\n"
+         "If the parameter inclusive=true, the definition of an 'interval' is exclusive of the left split point (or minimum value) and "
+         "inclusive of the right split point.\n"
          "It is not necessary to include either the min or max values in these split points.")
     .def("get_cdf", &dspy::req_sketch_get_cdf<T>, py::arg("split_points"), py::arg("inclusive")=false,
          "Returns an approximation to the Cumulative Distribution Function (CDF), which is the "
@@ -215,9 +220,11 @@ void bind_req_sketch(py::module &m, const char* name) {
          "If the sketch is empty this returns an empty vector.\n"
          "split_points is an array of m unique, monotonically increasing float values "
          "that divide the real number line into m+1 consecutive disjoint intervals.\n"
-         "The definition of an 'interval' is inclusive of the left split point (or minimum value) and "
+         "If the parameter inclusive=false, the definition of an 'interval' is inclusive of the left split point (or minimum value) and "
          "exclusive of the right split point, with the exception that the last interval will include "
          "the maximum value.\n"
+         "If the parameter inclusive=true, the definition of an 'interval' is exclusive of the left split point (or minimum value) and "
+         "inclusive of the right split point.\n"
          "It is not necessary to include either the min or max values in these split points.")
     .def("get_rank_lower_bound", &req_sketch<T>::get_rank_lower_bound, py::arg("rank"), py::arg("num_std_dev"),
          "Returns an approximate lower bound on the given normalized rank.\n"

data/vendor/datasketches-cpp/python/src/theta_wrapper.cpp

@@ -103,7 +103,7 @@ void init_theta(py::module &m) {
 
   py::class_<update_theta_sketch, theta_sketch>(m, "update_theta_sketch")
     .def(py::init(&dspy::update_theta_sketch_factory),
-         py::arg("lg_k")=update_theta_sketch::builder::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED)
+         py::arg("lg_k")=theta_constants::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED)
     .def(py::init<const update_theta_sketch&>())
     .def("update", (void (update_theta_sketch::*)(int64_t)) &update_theta_sketch::update, py::arg("datum"),
          "Updates the sketch with the given integral value")
@@ -127,7 +127,7 @@ void init_theta(py::module &m) {
 
   py::class_<theta_union>(m, "theta_union")
     .def(py::init(&dspy::theta_union_factory),
-         py::arg("lg_k")=update_theta_sketch::builder::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED)
+         py::arg("lg_k")=theta_constants::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED)
     .def("update", &theta_union::update<const theta_sketch&>, py::arg("sketch"),
          "Updates the union with the given sketch")
     .def("get_result", &theta_union::get_result, py::arg("ordered")=true,

data/vendor/datasketches-cpp/python/src/vector_of_kll.cpp

@@ -24,19 +24,26 @@
 #include <pybind11/numpy.h>
 #include <sstream>
 #include <vector>
+#include <stdexcept>
 
 namespace py = pybind11;
 
 namespace datasketches {
 
+namespace vector_of_kll_constants {
+  static const uint32_t DEFAULT_K = kll_constants::DEFAULT_K;
+  static const uint32_t DEFAULT_D = 1;
+}
+
 // Wrapper class for Numpy compatibility
 template <typename T, typename C = std::less<T>, typename S = serde<T>>
 class vector_of_kll_sketches {
   public:
-    static const uint32_t DEFAULT_K = kll_sketch<T, C, S>::DEFAULT_K;
-    static const uint32_t DEFAULT_D = 1;
+    // TODO: Redundant and deprecated. Will be removed in next major version release.
+    static const uint32_t DEFAULT_K = vector_of_kll_constants::DEFAULT_K;
+    static const uint32_t DEFAULT_D = vector_of_kll_constants::DEFAULT_D;
 
-    explicit vector_of_kll_sketches(uint32_t k = DEFAULT_K, uint32_t d = DEFAULT_D);
+    explicit vector_of_kll_sketches(uint32_t k = vector_of_kll_constants::DEFAULT_K, uint32_t d = vector_of_kll_constants::DEFAULT_D);
     vector_of_kll_sketches(const vector_of_kll_sketches& other);
     vector_of_kll_sketches(vector_of_kll_sketches&& other) noexcept;
     vector_of_kll_sketches<T,C,S>& operator=(const vector_of_kll_sketches& other);
@@ -432,8 +439,8 @@ void bind_vector_of_kll_sketches(py::module &m, const char* name) {
   using namespace datasketches;
 
   py::class_<vector_of_kll_sketches<T>>(m, name)
-    .def(py::init<uint32_t, uint32_t>(), py::arg("k")=vector_of_kll_sketches<T>::DEFAULT_K, 
-                                         py::arg("d")=vector_of_kll_sketches<T>::DEFAULT_D)
+    .def(py::init<uint32_t, uint32_t>(), py::arg("k")=vector_of_kll_constants::DEFAULT_K, 
+                                         py::arg("d")=vector_of_kll_constants::DEFAULT_D)
     .def(py::init<const vector_of_kll_sketches<T>&>())
     // allow user to retrieve k or d, in case it's instantiated w/ defaults
     .def("get_k", &vector_of_kll_sketches<T>::get_k,

data/vendor/datasketches-cpp/python/tests/kll_test.py

@@ -16,7 +16,7 @@
 # under the License.
 
 import unittest
-from datasketches import kll_ints_sketch, kll_floats_sketch
+from datasketches import kll_ints_sketch, kll_floats_sketch, kll_doubles_sketch, ks_test
 import numpy as np
 
 class KllTest(unittest.TestCase):
@@ -30,10 +30,10 @@ class KllTest(unittest.TestCase):
       kll.update(0.0)
 
       # 0 should be near the median
-      self.assertAlmostEqual(0.5, kll.get_rank(0.0), delta=0.025)
+      self.assertAlmostEqual(0.5, kll.get_rank(0.0), delta=0.035)
       
       # the median should be near 0
-      self.assertAlmostEqual(0.0, kll.get_quantile(0.5), delta=0.025)
+      self.assertAlmostEqual(0.0, kll.get_quantile(0.5), delta=0.035)
 
       # we also track the min/max independently from the rest of the data
       # which lets us know the full observed data range
@@ -73,6 +73,12 @@ class KllTest(unittest.TestCase):
       self.assertEqual(kll.get_quantile(0.7), new_kll.get_quantile(0.7))
       self.assertEqual(kll.get_rank(0.0), new_kll.get_rank(0.0))
 
+      # A Kolmogorov-Smirnov Test of kll and new_kll should match, even for
+      # a fairly small p-value -- cannot reject the null hypothesis that
+      # they come from the same distribution (since they do)
+      self.assertFalse(ks_test(kll, new_kll, 0.001))
+
+
     def test_kll_ints_sketch(self):
         k = 100
         n = 10
@@ -109,10 +115,10 @@ class KllTest(unittest.TestCase):
         sk_bytes = kll.serialize()
         self.assertTrue(isinstance(kll_ints_sketch.deserialize(sk_bytes), kll_ints_sketch))
 
-    def test_kll_floats_sketch(self):
-      # already tested ints and it's templatized, so just make sure it instantiates properly
+    def test_kll_doubles_sketch(self):
+      # already tested float and ints and it's templatized, so just make sure it instantiates properly
       k = 75
-      kll = kll_floats_sketch(k)
+      kll = kll_doubles_sketch(k)
       self.assertTrue(kll.is_empty())
 
 if __name__ == '__main__':