datasketches 0.2.0 → 0.2.4

data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp

@@ -70,12 +70,10 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch.get_PMF(split_points, 1).size() == 0);
     REQUIRE(sketch.get_CDF(split_points, 1).size() == 0);
 
-    int count = 0;
-    for (auto& it: sketch) {
+    for (auto it: sketch) {
       (void) it; // to suppress "unused" warning
-      ++count;
+      FAIL("should be no iterations over an empty sketch");
     }
-    REQUIRE(count == 0);
 }
 
   SECTION("get bad quantile") {
@@ -86,13 +84,13 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
   SECTION("one item") {
     kll_float_sketch sketch(200, 0);
-    sketch.update(1);
+    sketch.update(1.0f);
     REQUIRE_FALSE(sketch.is_empty());
     REQUIRE_FALSE(sketch.is_estimation_mode());
     REQUIRE(sketch.get_n() == 1);
     REQUIRE(sketch.get_num_retained() == 1);
-    REQUIRE(sketch.get_rank(1) == 0.0);
-    REQUIRE(sketch.get_rank(2) == 1.0);
+    REQUIRE(sketch.get_rank(1.0f) == 0.0);
+    REQUIRE(sketch.get_rank(2.0f) == 1.0);
     REQUIRE(sketch.get_min_value() == 1.0);
     REQUIRE(sketch.get_max_value() == 1.0);
     REQUIRE(sketch.get_quantile(0.5) == 1.0);
@@ -104,7 +102,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(quantiles[2] == 1.0);
 
     int count = 0;
-    for (auto& it: sketch) {
+    for (auto it: sketch) {
       REQUIRE(it.second == 1);
       ++count;
     }
@@ -116,16 +114,16 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     sketch.update(std::numeric_limits<float>::quiet_NaN());
     REQUIRE(sketch.is_empty());
 
-    sketch.update(0.0);
+    sketch.update(0);
     sketch.update(std::numeric_limits<float>::quiet_NaN());
     REQUIRE(sketch.get_n() == 1);
   }
 
   SECTION("many items, exact mode") {
     kll_float_sketch sketch(200, 0);
-    const uint32_t n(200);
+    const uint32_t n = 200;
     for (uint32_t i = 0; i < n; i++) {
-      sketch.update(i);
+      sketch.update(static_cast<float>(i));
       REQUIRE(sketch.get_n() == i + 1);
     }
     REQUIRE_FALSE(sketch.is_empty());
@@ -145,7 +143,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
     for (uint32_t i = 0; i < n; i++) {
       const double trueRank = (double) i / n;
-      REQUIRE(sketch.get_rank(i) == trueRank);
+      REQUIRE(sketch.get_rank(static_cast<float>(i)) == trueRank);
     }
 
     // the alternative method must produce the same result
@@ -158,16 +156,16 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
   SECTION("10 items") {
     kll_float_sketch sketch(200, 0);
-    sketch.update(1);
-    sketch.update(2);
-    sketch.update(3);
-    sketch.update(4);
-    sketch.update(5);
-    sketch.update(6);
-    sketch.update(7);
-    sketch.update(8);
-    sketch.update(9);
-    sketch.update(10);
+    sketch.update(1.0f);
+    sketch.update(2.0f);
+    sketch.update(3.0f);
+    sketch.update(4.0f);
+    sketch.update(5.0f);
+    sketch.update(6.0f);
+    sketch.update(7.0f);
+    sketch.update(8.0f);
+    sketch.update(9.0f);
+    sketch.update(10.0f);
     REQUIRE(sketch.get_quantile(0) == 1.0);
     REQUIRE(sketch.get_quantile(0.5) == 6.0);
     REQUIRE(sketch.get_quantile(0.99) == 10.0);
@@ -176,7 +174,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
   SECTION("100 items") {
     kll_float_sketch sketch(200, 0);
-    for (int i = 0; i < 100; ++i) sketch.update(i);
+    for (int i = 0; i < 100; ++i) sketch.update(static_cast<float>(i));
     REQUIRE(sketch.get_quantile(0) == 0);
     REQUIRE(sketch.get_quantile(0.01) == 1);
     REQUIRE(sketch.get_quantile(0.5) == 50);
@@ -186,9 +184,9 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
   SECTION("many items, estimation mode") {
     kll_float_sketch sketch(200, 0);
-    const int n(1000000);
+    const int n = 1000000;
     for (int i = 0; i < n; i++) {
-      sketch.update(i);
+      sketch.update(static_cast<float>(i));
       REQUIRE(sketch.get_n() == static_cast<uint64_t>(i + 1));
     }
     REQUIRE_FALSE(sketch.is_empty());
@@ -201,7 +199,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     // test rank
     for (int i = 0; i < n; i++) {
       const double trueRank = (double) i / n;
-      REQUIRE(sketch.get_rank(i) == Approx(trueRank).margin(RANK_EPS_FOR_K_200));
+      REQUIRE(sketch.get_rank(static_cast<float>(i)) == Approx(trueRank).margin(RANK_EPS_FOR_K_200));
     }
 
     // test quantiles at every 0.1 percentage point
@@ -224,6 +222,15 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     }
 
     //std::cout << sketch.to_string();
+
+    uint32_t count = 0;
+    uint64_t total_weight = 0;
+    for (auto it: sketch) {
+      ++count;
+      total_weight += it.second;
+    }
+    REQUIRE(count == sketch.get_num_retained());
+    REQUIRE(total_weight == sketch.get_n());
   }
 
   SECTION("consistency between get_rank adn get_PMF/CDF") {
@@ -231,8 +238,8 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     const int n = 1000;
     float values[n];
     for (int i = 0; i < n; i++) {
-      sketch.update(i);
-      values[i] = i;
+      sketch.update(static_cast<float>(i));
+      values[i] = static_cast<float>(i);
     }
 
     const auto ranks(sketch.get_CDF(values, n));
@@ -272,6 +279,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
     auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
+    REQUIRE(s.tellg() == s.tellp());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
     REQUIRE(sketch2.get_n() == sketch.get_n());
@@ -297,9 +305,9 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch2.get_normalized_rank_error(true) == sketch.get_normalized_rank_error(true));
   }
 
-  SECTION("serialize deserialize one item") {
+  SECTION("stream serialize deserialize one item") {
     kll_float_sketch sketch(200, 0);
-    sketch.update(1);
+    sketch.update(1.0f);
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
@@ -317,6 +325,24 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch2.get_rank(2) == 1.0);
   }
 
+  SECTION("bytes serialize deserialize one item") {
+    kll_float_sketch sketch(200, 0);
+    sketch.update(1.0f);
+    auto bytes = sketch.serialize();
+    REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
+    REQUIRE_FALSE(sketch2.is_empty());
+    REQUIRE_FALSE(sketch2.is_estimation_mode());
+    REQUIRE(sketch2.get_n() == 1);
+    REQUIRE(sketch2.get_num_retained() == 1);
+    REQUIRE(sketch2.get_min_value() == 1.0);
+    REQUIRE(sketch2.get_max_value() == 1.0);
+    REQUIRE(sketch2.get_quantile(0.5) == 1.0);
+    REQUIRE(sketch2.get_rank(1) == 0.0);
+    REQUIRE(sketch2.get_rank(2) == 1.0);
+  }
+
   SECTION("deserialize one item v1") {
     std::ifstream is;
     is.exceptions(std::ios::failbit | std::ios::badbit);
@@ -330,10 +356,46 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch.get_max_value() == 1.0);
   }
 
+  SECTION("stream serialize deserialize three items") {
+    kll_float_sketch sketch(200, 0);
+    sketch.update(1.0f);
+    sketch.update(2.0f);
+    sketch.update(3.0f);
+    std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+    sketch.serialize(s);
+    REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
+    auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
+    REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
+    REQUIRE(s.tellg() == s.tellp());
+    REQUIRE_FALSE(sketch2.is_empty());
+    REQUIRE_FALSE(sketch2.is_estimation_mode());
+    REQUIRE(sketch2.get_n() == 3);
+    REQUIRE(sketch2.get_num_retained() == 3);
+    REQUIRE(sketch2.get_min_value() == 1.0);
+    REQUIRE(sketch2.get_max_value() == 3.0);
+  }
+
+  SECTION("bytes serialize deserialize three items") {
+    kll_float_sketch sketch(200, 0);
+    sketch.update(1.0f);
+    sketch.update(2.0f);
+    sketch.update(3.0f);
+    auto bytes = sketch.serialize();
+    REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
+    REQUIRE_FALSE(sketch2.is_empty());
+    REQUIRE_FALSE(sketch2.is_estimation_mode());
+    REQUIRE(sketch2.get_n() == 3);
+    REQUIRE(sketch2.get_num_retained() == 3);
+    REQUIRE(sketch2.get_min_value() == 1.0);
+    REQUIRE(sketch2.get_max_value() == 3.0);
+  }
+
   SECTION("stream serialize deserialize many floats") {
     kll_float_sketch sketch(200, 0);
-    const int n(1000);
-    for (int i = 0; i < n; i++) sketch.update(i);
+    const int n = 1000;
+    for (int i = 0; i < n; i++) sketch.update(static_cast<float>(i));
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch.serialize(s);
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
@@ -350,13 +412,13 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch2.get_normalized_rank_error(true) == sketch.get_normalized_rank_error(true));
     REQUIRE(sketch2.get_quantile(0.5) == sketch.get_quantile(0.5));
     REQUIRE(sketch2.get_rank(0) == sketch.get_rank(0));
-    REQUIRE(sketch2.get_rank(n) == sketch.get_rank(n));
+    REQUIRE(sketch2.get_rank(static_cast<float>(n)) == sketch.get_rank(static_cast<float>(n)));
   }
 
   SECTION("bytes serialize deserialize many floats") {
     kll_float_sketch sketch(200, 0);
-    const int n(1000);
-    for (int i = 0; i < n; i++) sketch.update(i);
+    const int n = 1000;
+    for (int i = 0; i < n; i++) sketch.update(static_cast<float>(i));
     auto bytes = sketch.serialize();
     REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
     auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
@@ -371,7 +433,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch2.get_normalized_rank_error(true) == sketch.get_normalized_rank_error(true));
     REQUIRE(sketch2.get_quantile(0.5) == sketch.get_quantile(0.5));
     REQUIRE(sketch2.get_rank(0) == sketch.get_rank(0));
-    REQUIRE(sketch2.get_rank(n) == sketch.get_rank(n));
+    REQUIRE(sketch2.get_rank(static_cast<float>(n)) == sketch.get_rank(static_cast<float>(n)));
     REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(bytes.data(), 7), std::out_of_range);
     REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(bytes.data(), 15), std::out_of_range);
     REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(bytes.data(), bytes.size() - 1), std::out_of_range);
@@ -379,7 +441,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
   SECTION("bytes serialize deserialize many ints") {
     kll_sketch<int> sketch;
-    const int n(1000);
+    const int n = 1000;
     for (int i = 0; i < n; i++) sketch.update(i);
     auto bytes = sketch.serialize();
     REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
@@ -439,8 +501,8 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     kll_float_sketch sketch2(200, 0);
     const int n = 10000;
     for (int i = 0; i < n; i++) {
-      sketch1.update(i);
-      sketch2.update((2 * n) - i - 1);
+      sketch1.update(static_cast<float>(i));
+      sketch2.update(static_cast<float>((2 * n) - i - 1));
     }
 
     REQUIRE(sketch1.get_min_value() == 0.0f);
@@ -462,8 +524,8 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     kll_float_sketch sketch2(128, 0);
     const int n = 10000;
     for (int i = 0; i < n; i++) {
-      sketch1.update(i);
-      sketch2.update((2 * n) - i - 1);
+      sketch1.update(static_cast<float>(i));
+      sketch2.update(static_cast<float>((2 * n) - i - 1));
     }
 
     REQUIRE(sketch1.get_min_value() == 0.0f);
@@ -495,7 +557,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     kll_float_sketch sketch2(128, 0);
     const int n = 10000;
     for (int i = 0; i < n; i++) {
-      sketch1.update(i);
+      sketch1.update(static_cast<float>(i));
     }
 
     // rank error should not be affected by a merge with an empty sketch with lower k
@@ -518,8 +580,8 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   SECTION("merge min value from other") {
     kll_float_sketch sketch1(200, 0);
     kll_float_sketch sketch2(200, 0);
-    sketch1.update(1);
-    sketch2.update(2);
+    sketch1.update(1.0f);
+    sketch2.update(2.0f);
     sketch2.merge(sketch1);
     REQUIRE(sketch2.get_min_value() == 1.0f);
     REQUIRE(sketch2.get_max_value() == 2.0f);
@@ -527,7 +589,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
 
   SECTION("merge min and max values from other") {
     kll_float_sketch sketch1(200, 0);
-    for (int i = 0; i < 1000000; i++) sketch1.update(i);
+    for (int i = 0; i < 1000000; i++) sketch1.update(static_cast<float>(i));
     kll_float_sketch sketch2(200, 0);
     sketch2.merge(sketch1);
     REQUIRE(sketch2.get_min_value() == 0.0f);
@@ -540,7 +602,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE_THROWS_AS(sketch.get_min_value(), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_max_value(), std::runtime_error);
 
-    const int n(1000);
+    const int n = 1000;
     for (int i = 0; i < n; i++) sketch.update(i);
 
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
@@ -679,6 +741,31 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     }
   }
 
+  SECTION("max serialized size arithmetic type") {
+    REQUIRE(kll_sketch<float>::get_max_serialized_size_bytes(200, 10) == 1968);
+    REQUIRE(kll_sketch<float>::get_max_serialized_size_bytes(200, 100) == 2316);
+    REQUIRE(kll_sketch<float>::get_max_serialized_size_bytes(200, 1000) == 2440);
+    REQUIRE(kll_sketch<float>::get_max_serialized_size_bytes(200, 1000000) == 2800);
+    REQUIRE(kll_sketch<float>::get_max_serialized_size_bytes(200, 1000000000) == 3160);
+  }
+
+  SECTION("max serialized size non-arithmetic type") {
+    REQUIRE(kll_sketch<std::string>::get_max_serialized_size_bytes(200, 10, 4) == 1968);
+    REQUIRE(kll_sketch<std::string>::get_max_serialized_size_bytes(200, 100, 4) == 2316);
+    REQUIRE(kll_sketch<std::string>::get_max_serialized_size_bytes(200, 1000, 4) == 2440);
+    REQUIRE(kll_sketch<std::string>::get_max_serialized_size_bytes(200, 1000000, 4) == 2800);
+    REQUIRE(kll_sketch<std::string>::get_max_serialized_size_bytes(200, 1000000000, 4) == 3160);
+  }
+
+  SECTION("issue #236") {
+    kll_sketch<int8_t> kll;
+    kll.update(1);
+    kll.update(2);
+    kll.update(3);
+    auto blob = kll.serialize();
+    auto kll2 = kll_sketch<int8_t>::deserialize(blob.data(), blob.size());
+  }
+
   // cleanup
   if (test_allocator_total_bytes != 0) {
     REQUIRE(test_allocator_total_bytes == 0);

data/vendor/datasketches-cpp/kll/test/kolmogorov_smirnov_test.cpp

@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch.hpp>
+
+#include <random>
+
+#include <kll_sketch.hpp>
+#include <kolmogorov_smirnov.hpp>
+
+namespace datasketches {
+
+TEST_CASE("kolmogorov-smirnov empty", "[kll_sketch]") {
+  const uint16_t k = 200;
+  kll_sketch<double> sketch1(k);
+  kll_sketch<double> sketch2(k);
+  REQUIRE(kolmogorov_smirnov::delta(sketch1, sketch2) == 0);
+  REQUIRE_FALSE(kolmogorov_smirnov::test(sketch1, sketch2, 0.01));
+}
+
+TEST_CASE("kolmogorov-smirnov same distribution", "[kll_sketch]") {
+  const uint16_t k = 200;
+  kll_sketch<double> sketch1(k);
+  kll_sketch<double> sketch2(k);
+  std::default_random_engine rand;
+  std::normal_distribution<double> distr;
+  const int n = k * 3 - 1;
+  for (int i = 0; i < n; ++i) {
+    const double x = distr(rand);
+    sketch1.update(x);
+    sketch2.update(x);
+  }
+  REQUIRE(kolmogorov_smirnov::delta(sketch1, sketch2) == Approx(0).margin(0.01));
+  REQUIRE_FALSE(kolmogorov_smirnov::test(sketch1, sketch2, 0.01));
+}
+
+TEST_CASE("kolmogorov-smirnov very different distributions", "[kll_sketch]") {
+  const uint16_t k = 200;
+  kll_sketch<double> sketch1(k);
+  kll_sketch<double> sketch2(k);
+  std::default_random_engine rand;
+  std::normal_distribution<double> distr;
+  const int n = k * 3 - 1;
+  for (int i = 0; i < n; ++i) {
+    const double x = distr(rand);
+    sketch1.update(x + 100.0);
+    sketch2.update(x);
+  }
+  const auto delta = kolmogorov_smirnov::delta(sketch1, sketch2);
+  REQUIRE(delta == Approx(1.0).margin(1e-6));
+  REQUIRE(delta <= 1);
+  REQUIRE(kolmogorov_smirnov::test(sketch1, sketch2, 0.05));
+}
+
+TEST_CASE("kolmogorov-smirnov slightly different distributions", "[kll_sketch]") {
+  const uint16_t k = 2000;
+  kll_sketch<double> sketch1(k);
+  kll_sketch<double> sketch2(k);
+  std::default_random_engine rand;
+  std::normal_distribution<double> distr;
+  const int n = k * 3 - 1;
+  for (int i = 0; i < n; ++i) {
+    const double x = distr(rand);
+    sketch1.update(x + 0.05);
+    sketch2.update(x);
+  }
+  const double delta = kolmogorov_smirnov::delta(sketch1, sketch2);
+  REQUIRE(delta == Approx(0.02).margin(0.01));
+  const double threshold = kolmogorov_smirnov::threshold(sketch1, sketch2, 0.05);
+  //std::cout << "delta=" << delta << ", threshold=" << threshold << "\n";
+  REQUIRE_FALSE(delta > threshold);
+  REQUIRE_FALSE(kolmogorov_smirnov::test(sketch1, sketch2, 0.05));
+}
+
+TEST_CASE("kolmogorov-smirnov slightly different distributions high resolution", "[kll_sketch]") {
+  const uint16_t k = 8000;
+  kll_sketch<double> sketch1(k);
+  kll_sketch<double> sketch2(k);
+  std::default_random_engine rand;
+  std::normal_distribution<double> distr;
+  const int n = k * 3 - 1;
+  for (int i = 0; i < n; ++i) {
+    const double x = distr(rand);
+    sketch1.update(x + 0.05);
+    sketch2.update(x);
+  }
+  const double delta = kolmogorov_smirnov::delta(sketch1, sketch2);
+  REQUIRE(delta == Approx(0.02).margin(0.01));
+  const double threshold = kolmogorov_smirnov::threshold(sketch1, sketch2, 0.05);
+  //std::cout << "delta=" << delta << ", threshold=" << threshold << "\n";
+  REQUIRE(delta > threshold);
+  REQUIRE(kolmogorov_smirnov::test(sketch1, sketch2, 0.05));
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/pyproject.toml

@@ -1,8 +1,10 @@
 [build-system]
 requires = ["wheel",
             "setuptools >= 30.3.0",
-            "setuptools_scm",
-            "cmake >= 3.12"]
+            "cmake >= 3.12",
+            "pip >= 10.0",
+            "pybind11[global] >= 2.6.0"]
+build-backend = "setuptools.build_meta"
 
 [tool.tox]
 legacy_tox_ini = """

data/vendor/datasketches-cpp/python/CMakeLists.txt

@@ -15,16 +15,20 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# TODO: Can we force python version >= 3.0?
+find_package(Python3 COMPONENTS Interpreter Development)
+
+# only Windows+MSVC seems to have trouble locating pybind11
 if (MSVC)
-  set(PYBIND11_CPP_STANDARD /std:c++11)
-else()
-  set(PYBIND11_CPP_STANDARD -std=c++11)
+  execute_process(COMMAND cmd.exe /c ${CMAKE_CURRENT_SOURCE_DIR}/pybind11Path.cmd "${Python3_EXECUTABLE}"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      OUTPUT_VARIABLE EXTRA_PACKAGE_PATH)
+  set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${EXTRA_PACKAGE_PATH})
 endif()
 
-add_subdirectory(pybind11)
+find_package(pybind11 CONFIG REQUIRED)
 
-pybind11_add_module(python MODULE EXCLUDE_FROM_ALL SYSTEM THIN_LTO)
+pybind11_add_module(python MODULE EXCLUDE_FROM_ALL THIN_LTO)
 
 target_link_libraries(python
   PRIVATE

data/vendor/datasketches-cpp/python/README.md

@@ -1,76 +1,57 @@
-# Python Wrapper for Apache DataSketches
+<img src="https://raw.githubusercontent.com/apache/datasketches-website/master/logos/svg/datasketches-HorizontalColor-TM.svg" width="75%" alt="Apache DataSketchs Logo">
 
-## Installation
+# The Apache DataSketches Library for Python
 
-The release files do not include the needed python binding library ([pybind11](https://github.com/pybind/pybind11)). If building
-from a relase package, you must ensure that the pybind11 directory points to a local copy of pybind11.
+This is the official version of the [Apache DataSketches](https://datasketches.apache.org) Python library.
 
-An official pypi build is eventually planned but not yet available.
+In the analysis of big data there are often problem queries that don’t scale because they require huge compute resources and time to generate exact results. Examples include count distinct, quantiles, most-frequent items, joins, matrix computations, and graph analysis.
 
-If you instead want to take a (possibly ill-advised) gamble on the current state of the master branch being useable, you can run:
-```pip install git+https://github.com/apache/datasketches-cpp.git```
+If approximate results are acceptable, there is a class of specialized algorithms, called streaming algorithms, or sketches that can produce results orders-of magnitude faster and with mathematically proven error bounds. For interactive queries there may not be other viable alternatives, and in the case of real-time analysis, sketches are the only known solution.
 
-## Developer Instructions
-
-### Building
-
-When cloning the source repository, you should include the pybind11 submodule with the `--recursive` option to the clone command:
-```
-git clone --recursive https://github.com/apache/datasketches-cpp.git
-cd datasketches-cpp
-python -m pip install --upgrade pip setuptools wheel numpy
-python setup.py build
-```
+This package provides a variety of sketches as described below. Wherever a specific type of sketch exists in Apache DataSketches packages for other languages, the sketches will be portable between languages (for platforms with the same endianness).
 
-If you cloned without `--recursive`, you can add the submodule post-checkout using `git submodule update --init --recursive`.
+## Building and Installation
 
-### Installing
+Once cloned, the library can be installed by running `python -m pip install .` in the project root directory, which will also install the necessary dependencies, namely numpy and [pybind11[global]](https://github.com/pybind/pybind11).
 
-Assuming you have already checked out the library and any dependent submodules, install by simply replacing the lsat
-line of the build command with `python setup.py install`.
+If you prefer to call the `setup.py` build script directly, you must first install `pybind11[global]`, as well as any other dependencies listed under the build-system section in `pyproject.toml`.
 
-### Unit tests
-
-The python tests are run with `tox`. To ensure you have all the needed packages, from the package base directory run:
-```
-python -m pip install --upgrade pip setuptools wheel numpy tox
-tox
-```
+The library is also available from PyPI via `python -m pip install datasketches`.
 
 ## Usage
 
-Having installed the library, loading the Apache Datasketches library in Python is simple: `import datasketches`.
+Having installed the library, loading the Apache Datasketches Library in Python is simple: `import datasketches`.
 
 ## Available Sketch Classes
 
 - KLL (Absolute Error Quantiles)
-    - `kll_ints_sketch`
-    - `kll_floats_sketch`
+  - `kll_ints_sketch`
+  - `kll_floats_sketch`
 - REQ (Relative Error Quantiles)
-    - `req_ints_sketch`
-    - `req_floats_sketch`
+  - `req_ints_sketch`
+  - `req_floats_sketch`
 - Frequent Items
-    - `frequent_strings_sketch`
-    - Error types are `frequent_items_error_type.{NO_FALSE_NEGATIVES | NO_FALSE_POSITIVES}`
+  - `frequent_strings_sketch`
+  - Error types are `frequent_items_error_type.{NO_FALSE_NEGATIVES | NO_FALSE_POSITIVES}`
 - Theta
-    - `update_theta_sketch`
-    - `compact_theta_sketch` (cannot be instantiated directly)
-    - `theta_union`
-    - `theta_intersection`
-    - `theta_a_not_b`
+  - `update_theta_sketch`
+  - `compact_theta_sketch` (cannot be instantiated directly)
+  - `theta_union`
+  - `theta_intersection`
+  - `theta_a_not_b`
 - HLL
-    - `hll_sketch`
-    - `hll_union`
-    - Target HLL types are `tgt_hll_type.{HLL_4 | HLL_6 | HLL_8}`
+  - `hll_sketch`
+  - `hll_union`
+  - Target HLL types are `tgt_hll_type.{HLL_4 | HLL_6 | HLL_8}`
 - CPC
-    - `cpc_sketch`
-    - `cpc_union`
+  - `cpc_sketch`
+  - `cpc_union`
 - VarOpt Sampling
-    - `var_opt_sketch`
-    - `var_opt_union`
+  - `var_opt_sketch`
+  - `var_opt_union`
 - Vector of KLL
-    - `vector_of_kll_ints_sketches`
-    - `vector_of_kll_floats_sketches`
+  - `vector_of_kll_ints_sketches`
+  - `vector_of_kll_floats_sketches`
 
 ## Known Differences from C++
 
@@ -79,3 +60,22 @@ The Python API largely mirrors the C++ API, with a few minor exceptions: The pri
 The Vector of KLL object is currently exclusive to python, and holds an array of independent KLL sketches. This is useful for creating a set of KLL sketches over a vector and has been designed to allow input as either a vector or a matrix of multiple vectors.
 
 We have also removed reliance on a builder class for theta sketches as Python allows named arguments to the constructor, not strictly positional arguments.
+
+## Developer Instructions
+
+The only developer-specific instructions relate to running unit tests.
+
+### Unit tests
+
+The Python unit tests are run with `tox`. To ensure you have all the needed package, from the package base directory run:
+
+```bash
+python -m pip install --upgrade tox
+tox
+```
+
+## License
+
+The Apache DataSketches Library is distrubted under an Apache 2.0 License.
+
+There may be precompiled binaries provided as a convenience and distributed through PyPI via [https://pypi.org/project/datasketches/] contain compiled code from [pybind11](https://github.com/pybind/pybind11), which is distributed under a BSD license.

data/vendor/datasketches-cpp/python/pybind11Path.cmd

@@ -0,0 +1,3 @@
+@echo off
+:: Takes path to the Python interpreter and returns the path to pybind11
+%1 -m pip show pybind11 | %1 -c "import sys,re;[sys.stdout.write(re.sub('^Location:\\s+','',line)) for line in sys.stdin if re.search('^Location:\\s+',line)]"

data/vendor/datasketches-cpp/python/src/cpc_wrapper.cpp

@@ -53,7 +53,7 @@ void init_cpc(py::module &m) {
   using namespace datasketches;
 
   py::class_<cpc_sketch>(m, "cpc_sketch")
-    .def(py::init<uint8_t, uint64_t>(), py::arg("lg_k")=CPC_DEFAULT_LG_K, py::arg("seed")=DEFAULT_SEED)
+    .def(py::init<uint8_t, uint64_t>(), py::arg("lg_k")=cpc_constants::DEFAULT_LG_K, py::arg("seed")=DEFAULT_SEED)
     .def(py::init<const cpc_sketch&>())
     .def("__str__", &cpc_sketch::to_string,
          "Produces a string summary of the sketch")