datasketches 0.2.2 → 0.2.5

data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp

@@ -22,6 +22,7 @@
 #include <cstring>
 #include <sstream>
 #include <fstream>
+#include <stdexcept>
 
 #include <kll_sketch.hpp>
 #include <test_allocator.hpp>
@@ -90,7 +91,9 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch.get_n() == 1);
     REQUIRE(sketch.get_num_retained() == 1);
     REQUIRE(sketch.get_rank(1.0f) == 0.0);
+    REQUIRE(sketch.get_rank<true>(1.0f) == 1.0);
     REQUIRE(sketch.get_rank(2.0f) == 1.0);
+    REQUIRE(sketch.get_rank(std::numeric_limits<float>::infinity()) == 1.0);
     REQUIRE(sketch.get_min_value() == 1.0);
     REQUIRE(sketch.get_max_value() == 1.0);
     REQUIRE(sketch.get_quantile(0.5) == 1.0);
@@ -142,8 +145,10 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(quantiles[2] == n - 1 );
 
     for (uint32_t i = 0; i < n; i++) {
-      const double trueRank = (double) i / n;
-      REQUIRE(sketch.get_rank(static_cast<float>(i)) == trueRank);
+      const double true_rank = (double) i / n;
+      REQUIRE(sketch.get_rank(static_cast<float>(i)) == true_rank);
+      const double true_rank_inclusive = (double) (i + 1) / n;
+      REQUIRE(sketch.get_rank<true>(static_cast<float>(i)) == true_rank_inclusive);
     }
 
     // the alternative method must produce the same result
@@ -241,20 +246,38 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
       sketch.update(static_cast<float>(i));
       values[i] = static_cast<float>(i);
     }
-
-    const auto ranks(sketch.get_CDF(values, n));
-    const auto pmf(sketch.get_PMF(values, n));
-
-    double subtotal_pmf(0);
-    for (int i = 0; i < n; i++) {
-      if (sketch.get_rank(values[i]) != ranks[i]) {
-        std::cerr << "checking rank vs CDF for value " << i << std::endl;
-        REQUIRE(sketch.get_rank(values[i]) == ranks[i]);
+    { // inclusive=false (default)
+      const auto ranks(sketch.get_CDF(values, n));
+      const auto pmf(sketch.get_PMF(values, n));
+
+      double subtotal_pmf = 0;
+      for (int i = 0; i < n; i++) {
+        if (sketch.get_rank(values[i]) != ranks[i]) {
+          std::cerr << "checking rank vs CDF for value " << i << std::endl;
+          REQUIRE(sketch.get_rank(values[i]) == ranks[i]);
+        }
+        subtotal_pmf += pmf[i];
+        if (abs(ranks[i] - subtotal_pmf) > NUMERIC_NOISE_TOLERANCE) {
+          std::cerr << "CDF vs PMF for value " << i << std::endl;
+          REQUIRE(ranks[i] == Approx(subtotal_pmf).margin(NUMERIC_NOISE_TOLERANCE));
+        }
       }
-      subtotal_pmf += pmf[i];
-      if (abs(ranks[i] - subtotal_pmf) > NUMERIC_NOISE_TOLERANCE) {
-        std::cerr << "CDF vs PMF for value " << i << std::endl;
-        REQUIRE(ranks[i] == Approx(subtotal_pmf).margin(NUMERIC_NOISE_TOLERANCE));
+    }
+    {  // inclusive=true
+      const auto ranks(sketch.get_CDF<true>(values, n));
+      const auto pmf(sketch.get_PMF<true>(values, n));
+
+      double subtotal_pmf = 0;
+      for (int i = 0; i < n; i++) {
+        if (sketch.get_rank<true>(values[i]) != ranks[i]) {
+          std::cerr << "checking rank vs CDF for value " << i << std::endl;
+          REQUIRE(sketch.get_rank(values[i]) == ranks[i]);
+        }
+        subtotal_pmf += pmf[i];
+        if (abs(ranks[i] - subtotal_pmf) > NUMERIC_NOISE_TOLERANCE) {
+          std::cerr << "CDF vs PMF for value " << i << std::endl;
+          REQUIRE(ranks[i] == Approx(subtotal_pmf).margin(NUMERIC_NOISE_TOLERANCE));
+        }
       }
     }
   }
@@ -279,6 +302,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
     auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
+    REQUIRE(s.tellg() == s.tellp());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
     REQUIRE(sketch2.get_n() == sketch.get_n());
@@ -292,7 +316,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   SECTION("bytes serialize deserialize empty") {
     kll_float_sketch sketch(200, 0);
     auto bytes = sketch.serialize();
-    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde<float>(), 0);
     REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
@@ -304,13 +328,13 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch2.get_normalized_rank_error(true) == sketch.get_normalized_rank_error(true));
   }
 
-  SECTION("serialize deserialize one item") {
+  SECTION("stream serialize deserialize one item") {
     kll_float_sketch sketch(200, 0);
     sketch.update(1.0f);
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
+    auto sketch2 = kll_float_sketch::deserialize(s, serde<float>(), 0);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
     REQUIRE(s.tellg() == s.tellp());
     REQUIRE_FALSE(sketch2.is_empty());
@@ -324,11 +348,29 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch2.get_rank(2) == 1.0);
   }
 
+  SECTION("bytes serialize deserialize one item") {
+    kll_float_sketch sketch(200, 0);
+    sketch.update(1.0f);
+    auto bytes = sketch.serialize();
+    REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde<float>(), 0);
+    REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
+    REQUIRE_FALSE(sketch2.is_empty());
+    REQUIRE_FALSE(sketch2.is_estimation_mode());
+    REQUIRE(sketch2.get_n() == 1);
+    REQUIRE(sketch2.get_num_retained() == 1);
+    REQUIRE(sketch2.get_min_value() == 1.0);
+    REQUIRE(sketch2.get_max_value() == 1.0);
+    REQUIRE(sketch2.get_quantile(0.5) == 1.0);
+    REQUIRE(sketch2.get_rank(1) == 0.0);
+    REQUIRE(sketch2.get_rank(2) == 1.0);
+  }
+
   SECTION("deserialize one item v1") {
     std::ifstream is;
     is.exceptions(std::ios::failbit | std::ios::badbit);
     is.open(testBinaryInputPath + "kll_sketch_float_one_item_v1.sk", std::ios::binary);
-    auto sketch = kll_float_sketch::deserialize(is, test_allocator<float>(0));
+    auto sketch = kll_float_sketch::deserialize(is, serde<float>(), 0);
     REQUIRE_FALSE(sketch.is_empty());
     REQUIRE_FALSE(sketch.is_estimation_mode());
     REQUIRE(sketch.get_n() == 1);
@@ -337,6 +379,42 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch.get_max_value() == 1.0);
   }
 
+  SECTION("stream serialize deserialize three items") {
+    kll_float_sketch sketch(200, 0);
+    sketch.update(1.0f);
+    sketch.update(2.0f);
+    sketch.update(3.0f);
+    std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+    sketch.serialize(s);
+    REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
+    auto sketch2 = kll_float_sketch::deserialize(s, serde<float>(), 0);
+    REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
+    REQUIRE(s.tellg() == s.tellp());
+    REQUIRE_FALSE(sketch2.is_empty());
+    REQUIRE_FALSE(sketch2.is_estimation_mode());
+    REQUIRE(sketch2.get_n() == 3);
+    REQUIRE(sketch2.get_num_retained() == 3);
+    REQUIRE(sketch2.get_min_value() == 1.0);
+    REQUIRE(sketch2.get_max_value() == 3.0);
+  }
+
+  SECTION("bytes serialize deserialize three items") {
+    kll_float_sketch sketch(200, 0);
+    sketch.update(1.0f);
+    sketch.update(2.0f);
+    sketch.update(3.0f);
+    auto bytes = sketch.serialize();
+    REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde<float>(), 0);
+    REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
+    REQUIRE_FALSE(sketch2.is_empty());
+    REQUIRE_FALSE(sketch2.is_estimation_mode());
+    REQUIRE(sketch2.get_n() == 3);
+    REQUIRE(sketch2.get_num_retained() == 3);
+    REQUIRE(sketch2.get_min_value() == 1.0);
+    REQUIRE(sketch2.get_max_value() == 3.0);
+  }
+
   SECTION("stream serialize deserialize many floats") {
     kll_float_sketch sketch(200, 0);
     const int n = 1000;
@@ -344,7 +422,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
+    auto sketch2 = kll_float_sketch::deserialize(s, serde<float>(), 0);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
     REQUIRE(s.tellg() == s.tellp());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
@@ -366,7 +444,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     for (int i = 0; i < n; i++) sketch.update(static_cast<float>(i));
     auto bytes = sketch.serialize();
     REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
-    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde<float>(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
@@ -623,7 +701,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
     auto bytes = sketch1.serialize();
     REQUIRE(bytes.size() == sketch1.get_serialized_size_bytes());
-    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size(), serde<std::string>(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
     REQUIRE(sketch2.is_empty() == sketch1.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch1.is_estimation_mode());
@@ -644,7 +722,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     sketch1.update("a");
     auto bytes = sketch1.serialize();
     REQUIRE(bytes.size() == sketch1.get_serialized_size_bytes());
-    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    auto sketch2 = kll_string_sketch::deserialize(bytes.data(), bytes.size(), serde<std::string>(), 0);
     REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
   }
 
@@ -702,6 +780,61 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(kll_sketch<std::string>::get_max_serialized_size_bytes(200, 1000000000, 4) == 3160);
   }
 
+  SECTION("issue #236") {
+    kll_sketch<int8_t> kll;
+    kll.update(1);
+    kll.update(2);
+    kll.update(3);
+    auto blob = kll.serialize();
+    auto kll2 = kll_sketch<int8_t>::deserialize(blob.data(), blob.size());
+  }
+
+  SECTION("sorted view") {
+    kll_sketch<int> kll;
+    kll.update(2);
+    kll.update(3);
+    kll.update(1);
+
+    { // non-cumulative, using operator->
+      auto view = kll.get_sorted_view(false);
+      REQUIRE(view.size() == 3);
+      auto it = view.begin();
+      REQUIRE(it->first == 1);
+      REQUIRE(it->second == 1);
+      ++it;
+      REQUIRE(it->first == 2);
+      REQUIRE(it->second == 1);
+      ++it;
+      REQUIRE(it->first == 3);
+      REQUIRE(it->second == 1);
+    }
+    { // cumulative, non-inclusive, using operator->
+      auto view = kll.get_sorted_view(true);
+      REQUIRE(view.size() == 3);
+      auto it = view.begin();
+      REQUIRE(it->first == 1);
+      REQUIRE(it->second == 0);
+      ++it;
+      REQUIRE(it->first == 2);
+      REQUIRE(it->second == 1);
+      ++it;
+      REQUIRE(it->first == 3);
+      REQUIRE(it->second == 2);
+    }
+    { // cumulative, inclusive, using operator*
+      auto view = kll.get_sorted_view<true>(true);
+      REQUIRE(view.size() == 3);
+      auto it = view.begin();
+      REQUIRE((*it).first == 1);
+      REQUIRE((*it).second == 1);
+      ++it;
+      REQUIRE((*it).first == 2);
+      REQUIRE((*it).second == 2);
+      ++it;
+      REQUIRE((*it).first == 3);
+      REQUIRE((*it).second == 3);
+    }
+  }
   // cleanup
   if (test_allocator_total_bytes != 0) {
     REQUIRE(test_allocator_total_bytes == 0);

data/vendor/datasketches-cpp/kll/test/kolmogorov_smirnov_test.cpp

@@ -46,7 +46,7 @@ TEST_CASE("kolmogorov-smirnov same distribution", "[kll_sketch]") {
     sketch1.update(x);
     sketch2.update(x);
   }
-  REQUIRE(kolmogorov_smirnov::delta(sketch1, sketch2) == Approx(0).margin(0.01));
+  REQUIRE(kolmogorov_smirnov::delta(sketch1, sketch2) == Approx(0).margin(0.02));
   REQUIRE_FALSE(kolmogorov_smirnov::test(sketch1, sketch2, 0.01));
 }
 

data/vendor/datasketches-cpp/pyproject.toml

@@ -1,8 +1,10 @@
 [build-system]
 requires = ["wheel",
             "setuptools >= 30.3.0",
-            "setuptools_scm",
-            "cmake >= 3.12"]
+            "cmake >= 3.16",
+            "pip >= 10.0",
+            "pybind11[global] >= 2.6.0"]
+build-backend = "setuptools.build_meta"
 
 [tool.tox]
 legacy_tox_ini = """

data/vendor/datasketches-cpp/python/CMakeLists.txt

@@ -15,16 +15,24 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# TODO: Can we force python version >= 3.0?
-if (MSVC)
-  set(PYBIND11_CPP_STANDARD /std:c++11)
+if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18.0")
+find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED)
 else()
-  set(PYBIND11_CPP_STANDARD -std=c++11)
+find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
+endif()
+
+# only Windows+MSVC seems to have trouble locating pybind11
+if (MSVC)
+  execute_process(COMMAND cmd.exe /c ${CMAKE_CURRENT_SOURCE_DIR}/pybind11Path.cmd "${Python3_EXECUTABLE}"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      OUTPUT_VARIABLE EXTRA_PACKAGE_PATH)
+  set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${EXTRA_PACKAGE_PATH})
 endif()
 
-add_subdirectory(pybind11)
+find_package(pybind11 CONFIG REQUIRED)
 
-pybind11_add_module(python MODULE EXCLUDE_FROM_ALL SYSTEM THIN_LTO)
+pybind11_add_module(python MODULE EXCLUDE_FROM_ALL THIN_LTO)
 
 target_link_libraries(python
   PRIVATE
@@ -36,6 +44,7 @@ target_link_libraries(python
     theta
     sampling
     req
+    quantiles
     pybind11::module
 )
 
@@ -59,5 +68,7 @@ target_sources(python
     src/theta_wrapper.cpp
     src/vo_wrapper.cpp
     src/req_wrapper.cpp
+    src/quantiles_wrapper.cpp
+    src/ks_wrapper.cpp
     src/vector_of_kll.cpp
 )

data/vendor/datasketches-cpp/python/README.md

@@ -1,76 +1,64 @@
-# Python Wrapper for Apache DataSketches
+<img src="https://raw.githubusercontent.com/apache/datasketches-website/master/logos/svg/datasketches-HorizontalColor-TM.svg" width="75%" alt="Apache DataSketchs Logo">
 
-## Installation
+# The Apache DataSketches Library for Python
 
-The release files do not include the needed python binding library ([pybind11](https://github.com/pybind/pybind11)). If building
-from a relase package, you must ensure that the pybind11 directory points to a local copy of pybind11.
+This is the official version of the [Apache DataSketches](https://datasketches.apache.org) Python library.
 
-An official pypi build is eventually planned but not yet available.
+In the analysis of big data there are often problem queries that don’t scale because they require huge compute resources and time to generate exact results. Examples include count distinct, quantiles, most-frequent items, joins, matrix computations, and graph analysis.
 
-If you instead want to take a (possibly ill-advised) gamble on the current state of the master branch being useable, you can run:
-```pip install git+https://github.com/apache/datasketches-cpp.git```
+If approximate results are acceptable, there is a class of specialized algorithms, called streaming algorithms, or sketches that can produce results orders-of magnitude faster and with mathematically proven error bounds. For interactive queries there may not be other viable alternatives, and in the case of real-time analysis, sketches are the only known solution.
 
-## Developer Instructions
-
-### Building
-
-When cloning the source repository, you should include the pybind11 submodule with the `--recursive` option to the clone command:
-```
-git clone --recursive https://github.com/apache/datasketches-cpp.git
-cd datasketches-cpp
-python -m pip install --upgrade pip setuptools wheel numpy
-python setup.py build
-```
+This package provides a variety of sketches as described below. Wherever a specific type of sketch exists in Apache DataSketches packages for other languages, the sketches will be portable between languages (for platforms with the same endianness).
 
-If you cloned without `--recursive`, you can add the submodule post-checkout using `git submodule update --init --recursive`.
+## Building and Installation
 
-### Installing
+Once cloned, the library can be installed by running `python -m pip install .` in the project root directory, which will also install the necessary dependencies, namely numpy and [pybind11[global]](https://github.com/pybind/pybind11).
 
-Assuming you have already checked out the library and any dependent submodules, install by simply replacing the lsat
-line of the build command with `python setup.py install`.
+If you prefer to call the `setup.py` build script directly, you must first install `pybind11[global]`, as well as any other dependencies listed under the build-system section in `pyproject.toml`.
 
-### Unit tests
-
-The python tests are run with `tox`. To ensure you have all the needed packages, from the package base directory run:
-```
-python -m pip install --upgrade pip setuptools wheel numpy tox
-tox
-```
+The library is also available from PyPI via `python -m pip install datasketches`.
 
 ## Usage
 
-Having installed the library, loading the Apache Datasketches library in Python is simple: `import datasketches`.
+Having installed the library, loading the Apache Datasketches Library in Python is simple: `import datasketches`.
 
 ## Available Sketch Classes
 
 - KLL (Absolute Error Quantiles)
-    - `kll_ints_sketch`
-    - `kll_floats_sketch`
+  - `kll_ints_sketch`
+  - `kll_floats_sketch`
+  - `kll_doubles_sketch`
+- Quantiles (Absolute Error Quantiles, inferior algorithm)
+  - `quantiles_ints_sketch`
+  - `quantiles_floats_sketch`
+  - `quantiles_doubles_sketch`
 - REQ (Relative Error Quantiles)
-    - `req_ints_sketch`
-    - `req_floats_sketch`
+  - `req_ints_sketch`
+  - `req_floats_sketch`
 - Frequent Items
-    - `frequent_strings_sketch`
-    - Error types are `frequent_items_error_type.{NO_FALSE_NEGATIVES | NO_FALSE_POSITIVES}`
+  - `frequent_strings_sketch`
+  - Error types are `frequent_items_error_type.{NO_FALSE_NEGATIVES | NO_FALSE_POSITIVES}`
 - Theta
-    - `update_theta_sketch`
-    - `compact_theta_sketch` (cannot be instantiated directly)
-    - `theta_union`
-    - `theta_intersection`
-    - `theta_a_not_b`
+  - `update_theta_sketch`
+  - `compact_theta_sketch` (cannot be instantiated directly)
+  - `theta_union`
+  - `theta_intersection`
+  - `theta_a_not_b`
 - HLL
-    - `hll_sketch`
-    - `hll_union`
-    - Target HLL types are `tgt_hll_type.{HLL_4 | HLL_6 | HLL_8}`
+  - `hll_sketch`
+  - `hll_union`
+  - Target HLL types are `tgt_hll_type.{HLL_4 | HLL_6 | HLL_8}`
 - CPC
-    - `cpc_sketch`
-    - `cpc_union`
+  - `cpc_sketch`
+  - `cpc_union`
 - VarOpt Sampling
-    - `var_opt_sketch`
-    - `var_opt_union`
+  - `var_opt_sketch`
+  - `var_opt_union`
 - Vector of KLL
-    - `vector_of_kll_ints_sketches`
-    - `vector_of_kll_floats_sketches`
+  - `vector_of_kll_ints_sketches`
+  - `vector_of_kll_floats_sketches`
+- Kolmogorov-Smirnov Test
+  - `ks_test` applied to a pair of matched-type Absolute Error quantiles sketches
 
 ## Known Differences from C++
 
@@ -79,3 +67,22 @@ The Python API largely mirrors the C++ API, with a few minor exceptions: The pri
 The Vector of KLL object is currently exclusive to python, and holds an array of independent KLL sketches. This is useful for creating a set of KLL sketches over a vector and has been designed to allow input as either a vector or a matrix of multiple vectors.
 
 We have also removed reliance on a builder class for theta sketches as Python allows named arguments to the constructor, not strictly positional arguments.
+
+## Developer Instructions
+
+The only developer-specific instructions relate to running unit tests.
+
+### Unit tests
+
+The Python unit tests are run with `tox`. To ensure you have all the needed package, from the package base directory run:
+
+```bash
+python -m pip install --upgrade tox
+tox
+```
+
+## License
+
+The Apache DataSketches Library is distrubted under an Apache 2.0 License.
+
+There may be precompiled binaries provided as a convenience and distributed through PyPI via [https://pypi.org/project/datasketches/] contain compiled code from [pybind11](https://github.com/pybind/pybind11), which is distributed under a BSD license.

data/vendor/datasketches-cpp/python/pybind11Path.cmd

@@ -0,0 +1,3 @@
+@echo off
+:: Takes path to the Python interpreter and returns the path to pybind11
+%1 -m pip show pybind11 | %1 -c "import sys,re;[sys.stdout.write(re.sub('^Location:\\s+','',line)) for line in sys.stdin if re.search('^Location:\\s+',line)]"

data/vendor/datasketches-cpp/python/src/cpc_wrapper.cpp

@@ -53,7 +53,7 @@ void init_cpc(py::module &m) {
   using namespace datasketches;
 
   py::class_<cpc_sketch>(m, "cpc_sketch")
-    .def(py::init<uint8_t, uint64_t>(), py::arg("lg_k")=CPC_DEFAULT_LG_K, py::arg("seed")=DEFAULT_SEED)
+    .def(py::init<uint8_t, uint64_t>(), py::arg("lg_k")=cpc_constants::DEFAULT_LG_K, py::arg("seed")=DEFAULT_SEED)
     .def(py::init<const cpc_sketch&>())
     .def("__str__", &cpc_sketch::to_string,
          "Produces a string summary of the sketch")

data/vendor/datasketches-cpp/python/src/datasketches.cpp

@@ -28,6 +28,8 @@ void init_cpc(py::module& m);
 void init_theta(py::module& m);
 void init_vo(py::module& m);
 void init_req(py::module& m);
+void init_quantiles(py::module& m);
+void init_kolmogorov_smirnov(py::module& m);
 void init_vector_of_kll(py::module& m);
 
 PYBIND11_MODULE(datasketches, m) {
@@ -38,5 +40,7 @@ PYBIND11_MODULE(datasketches, m) {
   init_theta(m);
   init_vo(m);
   init_req(m);
+  init_quantiles(m);
+  init_kolmogorov_smirnov(m);
   init_vector_of_kll(m);
 }

data/vendor/datasketches-cpp/python/src/fi_wrapper.cpp

@@ -64,6 +64,11 @@ py::list fi_sketch_get_frequent_items(const frequent_items_sketch<T>& sk,
   return list;
 }
 
+template<typename T>
+size_t fi_sketch_get_serialized_size_bytes(const frequent_items_sketch<T>& sk) {
+  return sk.get_serialized_size_bytes();
+}
+
 }
 }
 
@@ -104,7 +109,7 @@ void bind_fi_sketch(py::module &m, const char* name) {
          "Returns the epsilon value used to compute a priori error for a given log2(max_map_size)")
     .def_static("get_apriori_error", &frequent_items_sketch<T>::get_apriori_error, py::arg("lg_max_map_size"), py::arg("estimated_total_weight"),
          "Returns the estimated a priori error given the max_map_size for the sketch and the estimated_total_stream_weight.")
-    .def("get_serialized_size_bytes", &frequent_items_sketch<T>::get_serialized_size_bytes,
+    .def("get_serialized_size_bytes", &dspy::fi_sketch_get_serialized_size_bytes<T>,
          "Computes the size needed to serialize the current state of the sketch. This can be expensive since every item needs to be looked at.")
     .def("serialize", &dspy::fi_sketch_serialize<T>, "Serializes the sketch into a bytes object")
     .def_static("deserialize", &dspy::fi_sketch_deserialize<T>, "Reads a bytes object and returns the corresponding frequent_strings_sketch")