datasketches 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c578044053c564421893cc4433f7fe557f23ba9d8a1995fc2a2c5f07742721a
-  data.tar.gz: f4122bd75e19fede015b01a5e5ad8e6130f75babe9c9160cc56f378480a16cee
+  metadata.gz: ea00e444de6dc1bebc2b8cf878a250f08717d55eaa55f63f6bec28f4be2af00d
+  data.tar.gz: 161b9089e3b8d0dbd99cfb6cc0af463934c42ba85f4788a08306369966f28571
 SHA512:
-  metadata.gz: 2d7c4d7306f28356557a816a78033b909561ccd8f843281a2b756e88cbdcb9936da7995ff80871a19e229675ead812aca00d6c639d63a6532998c3c1b35aa953
-  data.tar.gz: fdf0fe1d14e04bfddef9df1ae7958f6571a7f689865aa02e81713d4b250afeeeb8c90a168ce855728d9c831aff6d3ea71df91c71b7269a760c19488c42c92658
+  metadata.gz: '09eede1e6e4c0fe57c0116c4e8873670192fea845783687ca34890bd9358af9dd19a535774ab7dd667055cf6acd0d3913f044dcf2274e0ec092b33307250a74a'
+  data.tar.gz: b8bcaeb7af0d27e836f21941663229a2750922914c4f31f4ffbd6e3c3876320f9ce92916eb9730e02227b87e8f244bc08e5bc38541bf4ef4e3485203fff01942

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.3 (2021-09-29)
+
+- Updated DataSketches to 3.2.0
+
 ## 0.2.2 (2021-07-17)
 
 - Updated DataSketches to 3.1.0

data/LICENSE

@@ -284,11 +284,48 @@ APPENDIX B: Additional licenses relevant to this product.
     ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
     DEALINGS IN THE SOFTWARE.
     -------------------------------------------------------------
-    Code Locations
+    Code Locations:
       * https://github.com/apache/datasketches-cpp/blob/master/common/test/catch.hpp
     that is adapted from the above.
 
 
+    =============================================================
+    BSD License
+    =============================================================
+    Original source code:
+       https://github.com/pybind/pybind11/blob/master/LICENSE
+
+    Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch>, All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice, this
+       list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright notice,
+       this list of conditions and the following disclaimer in the documentation
+       and/or other materials provided with the distribution.
+
+    3. Neither the name of the copyright holder nor the names of its contributors
+       may be used to endorse or promote products derived from this software
+       without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+    ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+    OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+    -------------------------------------------------------------
+    Code Locations:
+    Found only in the convenience binaries distributed from PyPI, which rely
+    on pybind11 code during compilation.
+
 
     =============================================================
     Public Domain
@@ -297,7 +334,7 @@ APPENDIX B: Additional licenses relevant to this product.
        https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp
        Placed in the Public Domain by Austin Appleby
 
-    Code Locations
+    Code Locations:
       common/include/MurmurHash3.h
     that is adapted from the above.
     -------------------------------------------------------------
@@ -305,6 +342,6 @@ APPENDIX B: Additional licenses relevant to this product.
       * https://graphics.stanford.edu/~seander/bithacks.html
       * Placed in the Public Domain by Sean Eron Anderson
 
-    Code Locations
+    Code Locations:
       * common/include/ceiling_power_of_2.hpp
     that is adapted from the above.

data/NOTICE

@@ -1,5 +1,5 @@
 Apache DataSketches-cpp
-Copyright 2020 The Apache Software Foundation
+Copyright 2020-2021 The Apache Software Foundation
 
 Copyright 2015-2018 Yahoo
 Copyright 2019 Verizon Media

data/lib/datasketches/version.rb

@@ -1,3 +1,3 @@
 module DataSketches
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

data/vendor/datasketches-cpp/CMakeLists.txt

@@ -35,6 +35,8 @@ set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 #set(CMAKE_VERBOSE_MAKEFILE ON)
 set(CMAKE_MACOSX_RPATH ON)
 
+set(CMAKE_CXX_STANDARD 11)
+
 # enable compiler warnings globally
 # derived from https://foonathan.net/blog/2018/10/17/cmake-warnings.html
 # and https://arne-mertz.de/2018/07/cmake-properties-options/

data/vendor/datasketches-cpp/LICENSE

@@ -284,11 +284,48 @@ APPENDIX B: Additional licenses relevant to this product.
     ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
     DEALINGS IN THE SOFTWARE.
     -------------------------------------------------------------
-    Code Locations
+    Code Locations:
       * https://github.com/apache/datasketches-cpp/blob/master/common/test/catch.hpp
     that is adapted from the above.
 
 
+    =============================================================
+    BSD License
+    =============================================================
+    Original source code:
+       https://github.com/pybind/pybind11/blob/master/LICENSE
+
+    Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch>, All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice, this
+       list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright notice,
+       this list of conditions and the following disclaimer in the documentation
+       and/or other materials provided with the distribution.
+
+    3. Neither the name of the copyright holder nor the names of its contributors
+       may be used to endorse or promote products derived from this software
+       without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+    ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+    OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+    -------------------------------------------------------------
+    Code Locations:
+    Found only in the convenience binaries distributed from PyPI, which rely
+    on pybind11 code during compilation.
+
 
     =============================================================
     Public Domain
@@ -297,7 +334,7 @@ APPENDIX B: Additional licenses relevant to this product.
        https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp
        Placed in the Public Domain by Austin Appleby
     
-    Code Locations
+    Code Locations:
       common/include/MurmurHash3.h
     that is adapted from the above.
     -------------------------------------------------------------
@@ -305,7 +342,7 @@ APPENDIX B: Additional licenses relevant to this product.
       * https://graphics.stanford.edu/~seander/bithacks.html
       * Placed in the Public Domain by Sean Eron Anderson
       
-    Code Locations
+    Code Locations:
       * common/include/ceiling_power_of_2.hpp
     that is adapted from the above.
   

data/vendor/datasketches-cpp/NOTICE

@@ -1,5 +1,5 @@
 Apache DataSketches-cpp
-Copyright 2020 The Apache Software Foundation
+Copyright 2020-2021 The Apache Software Foundation
 
 Copyright 2015-2018 Yahoo
 Copyright 2019 Verizon Media

data/vendor/datasketches-cpp/common/include/common_defs.hpp

@@ -29,6 +29,8 @@ namespace datasketches {
 
 static const uint64_t DEFAULT_SEED = 9001;
 
+enum resize_factor { X1 = 0, X2, X4, X8 };
+
 template<typename A> using AllocChar = typename std::allocator_traits<A>::template rebind_alloc<char>;
 template<typename A> using string = std::basic_string<char, std::char_traits<char>, AllocChar<A>>;
 

data/vendor/datasketches-cpp/cpc/include/cpc_common.hpp

@@ -26,9 +26,16 @@
 
 namespace datasketches {
 
-static const uint8_t CPC_MIN_LG_K = 4;
-static const uint8_t CPC_MAX_LG_K = 26;
-static const uint8_t CPC_DEFAULT_LG_K = 11;
+namespace cpc_constants {
+    const uint8_t MIN_LG_K = 4;
+    const uint8_t MAX_LG_K = 26;
+    const uint8_t DEFAULT_LG_K = 11;
+}
+
+// TODO: Redundant and deprecated. Will be removed in next major version release.
+static const uint8_t CPC_MIN_LG_K = cpc_constants::MIN_LG_K;
+static const uint8_t CPC_MAX_LG_K = cpc_constants::MAX_LG_K;
+static const uint8_t CPC_DEFAULT_LG_K = cpc_constants::DEFAULT_LG_K;
 
 template<typename A> using AllocU8 = typename std::allocator_traits<A>::template rebind_alloc<uint8_t>;
 template<typename A> using AllocU16 = typename std::allocator_traits<A>::template rebind_alloc<uint16_t>;

data/vendor/datasketches-cpp/cpc/include/cpc_sketch.hpp

@@ -67,7 +67,7 @@ public:
    * @param lg_k base 2 logarithm of the number of bins in the sketch
    * @param seed for hash function
    */
-  explicit cpc_sketch_alloc(uint8_t lg_k = CPC_DEFAULT_LG_K, uint64_t seed = DEFAULT_SEED, const A& allocator = A());
+  explicit cpc_sketch_alloc(uint8_t lg_k = cpc_constants::DEFAULT_LG_K, uint64_t seed = DEFAULT_SEED, const A& allocator = A());
 
   using allocator_type = A;
   A get_allocator() const;

data/vendor/datasketches-cpp/cpc/include/cpc_union.hpp

@@ -45,7 +45,7 @@ public:
    * @param lg_k base 2 logarithm of the number of bins in the sketch
    * @param seed for hash function
    */
-  explicit cpc_union_alloc(uint8_t lg_k = CPC_DEFAULT_LG_K, uint64_t seed = DEFAULT_SEED, const A& allocator = A());
+  explicit cpc_union_alloc(uint8_t lg_k = cpc_constants::DEFAULT_LG_K, uint64_t seed = DEFAULT_SEED, const A& allocator = A());
 
   cpc_union_alloc(const cpc_union_alloc<A>& other);
   cpc_union_alloc(cpc_union_alloc<A>&& other) noexcept;

data/vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp

@@ -350,7 +350,7 @@ frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>:
   check_serial_version(serial_version);
   check_family_id(family_id);
   check_size(lg_cur_size, lg_max_size);
-  ensure_minimum_memory(size, 1ULL << preamble_longs);
+  ensure_minimum_memory(size, preamble_longs * sizeof(uint64_t));
 
   frequent_items_sketch<T, W, H, E, S, A> sketch(lg_max_size, lg_cur_size, allocator);
   if (!is_empty) {

data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp

@@ -153,6 +153,10 @@ template<typename A> using vector_u32 = std::vector<uint32_t, AllocU32<A>>;
 template<typename A> using AllocD = typename std::allocator_traits<A>::template rebind_alloc<double>;
 template<typename A> using vector_d = std::vector<double, AllocD<A>>;
 
+namespace kll_constants {
+  const uint16_t DEFAULT_K = 200;
+}
+
 template <typename T, typename C = std::less<T>, typename S = serde<T>, typename A = std::allocator<T>>
 class kll_sketch {
   public:
@@ -160,11 +164,12 @@ class kll_sketch {
     using comparator = C;
 
     static const uint8_t DEFAULT_M = 8;
-    static const uint16_t DEFAULT_K = 200;
+    // TODO: Redundant and deprecated. Will be remove din next major version.
+    static const uint16_t DEFAULT_K = kll_constants::DEFAULT_K;
     static const uint16_t MIN_K = DEFAULT_M;
     static const uint16_t MAX_K = (1 << 16) - 1;
 
-    explicit kll_sketch(uint16_t k = DEFAULT_K, const A& allocator = A());
+    explicit kll_sketch(uint16_t k = kll_constants::DEFAULT_K, const A& allocator = A());
     kll_sketch(const kll_sketch& other);
     kll_sketch(kll_sketch&& other) noexcept;
     ~kll_sketch();

data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp

@@ -575,7 +575,7 @@ kll_sketch<T, C, S, A> kll_sketch<T, C, S, A>::deserialize(const void* bytes, si
   check_preamble_ints(preamble_ints, flags_byte);
   check_serial_version(serial_version);
   check_family_id(family_id);
-  ensure_minimum_memory(size, 1ULL << preamble_ints);
+  ensure_minimum_memory(size, preamble_ints * sizeof(uint32_t));
 
   const bool is_empty(flags_byte & (1 << flags::IS_EMPTY));
   if (is_empty) return kll_sketch<T, C, S, A>(k, allocator);

data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp

@@ -279,6 +279,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
     auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
     REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
+    REQUIRE(s.tellg() == s.tellp());
     REQUIRE(sketch2.is_empty() == sketch.is_empty());
     REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
     REQUIRE(sketch2.get_n() == sketch.get_n());
@@ -304,7 +305,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch2.get_normalized_rank_error(true) == sketch.get_normalized_rank_error(true));
   }
 
-  SECTION("serialize deserialize one item") {
+  SECTION("stream serialize deserialize one item") {
     kll_float_sketch sketch(200, 0);
     sketch.update(1.0f);
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
@@ -324,6 +325,24 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch2.get_rank(2) == 1.0);
   }
 
+  SECTION("bytes serialize deserialize one item") {
+    kll_float_sketch sketch(200, 0);
+    sketch.update(1.0f);
+    auto bytes = sketch.serialize();
+    REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
+    REQUIRE_FALSE(sketch2.is_empty());
+    REQUIRE_FALSE(sketch2.is_estimation_mode());
+    REQUIRE(sketch2.get_n() == 1);
+    REQUIRE(sketch2.get_num_retained() == 1);
+    REQUIRE(sketch2.get_min_value() == 1.0);
+    REQUIRE(sketch2.get_max_value() == 1.0);
+    REQUIRE(sketch2.get_quantile(0.5) == 1.0);
+    REQUIRE(sketch2.get_rank(1) == 0.0);
+    REQUIRE(sketch2.get_rank(2) == 1.0);
+  }
+
   SECTION("deserialize one item v1") {
     std::ifstream is;
     is.exceptions(std::ios::failbit | std::ios::badbit);
@@ -337,6 +356,42 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch.get_max_value() == 1.0);
   }
 
+  SECTION("stream serialize deserialize three items") {
+    kll_float_sketch sketch(200, 0);
+    sketch.update(1.0f);
+    sketch.update(2.0f);
+    sketch.update(3.0f);
+    std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+    sketch.serialize(s);
+    REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
+    auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
+    REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
+    REQUIRE(s.tellg() == s.tellp());
+    REQUIRE_FALSE(sketch2.is_empty());
+    REQUIRE_FALSE(sketch2.is_estimation_mode());
+    REQUIRE(sketch2.get_n() == 3);
+    REQUIRE(sketch2.get_num_retained() == 3);
+    REQUIRE(sketch2.get_min_value() == 1.0);
+    REQUIRE(sketch2.get_max_value() == 3.0);
+  }
+
+  SECTION("bytes serialize deserialize three items") {
+    kll_float_sketch sketch(200, 0);
+    sketch.update(1.0f);
+    sketch.update(2.0f);
+    sketch.update(3.0f);
+    auto bytes = sketch.serialize();
+    REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
+    auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
+    REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
+    REQUIRE_FALSE(sketch2.is_empty());
+    REQUIRE_FALSE(sketch2.is_estimation_mode());
+    REQUIRE(sketch2.get_n() == 3);
+    REQUIRE(sketch2.get_num_retained() == 3);
+    REQUIRE(sketch2.get_min_value() == 1.0);
+    REQUIRE(sketch2.get_max_value() == 3.0);
+  }
+
   SECTION("stream serialize deserialize many floats") {
     kll_float_sketch sketch(200, 0);
     const int n = 1000;
@@ -702,6 +757,15 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(kll_sketch<std::string>::get_max_serialized_size_bytes(200, 1000000000, 4) == 3160);
   }
 
+  SECTION("issue #236") {
+    kll_sketch<int8_t> kll;
+    kll.update(1);
+    kll.update(2);
+    kll.update(3);
+    auto blob = kll.serialize();
+    auto kll2 = kll_sketch<int8_t>::deserialize(blob.data(), blob.size());
+  }
+
   // cleanup
   if (test_allocator_total_bytes != 0) {
     REQUIRE(test_allocator_total_bytes == 0);

data/vendor/datasketches-cpp/pyproject.toml

@@ -1,8 +1,10 @@
 [build-system]
 requires = ["wheel",
             "setuptools >= 30.3.0",
-            "setuptools_scm",
-            "cmake >= 3.12"]
+            "cmake >= 3.12",
+            "pip >= 10.0",
+            "pybind11[global] >= 2.6.0"]
+build-backend = "setuptools.build_meta"
 
 [tool.tox]
 legacy_tox_ini = """

data/vendor/datasketches-cpp/python/CMakeLists.txt

@@ -15,16 +15,20 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# TODO: Can we force python version >= 3.0?
+find_package(Python3 COMPONENTS Interpreter Development)
+
+# only Windows+MSVC seems to have trouble locating pybind11
 if (MSVC)
-  set(PYBIND11_CPP_STANDARD /std:c++11)
-else()
-  set(PYBIND11_CPP_STANDARD -std=c++11)
+  execute_process(COMMAND cmd.exe /c ${CMAKE_CURRENT_SOURCE_DIR}/pybind11Path.cmd "${Python3_EXECUTABLE}"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      OUTPUT_VARIABLE EXTRA_PACKAGE_PATH)
+  set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${EXTRA_PACKAGE_PATH})
 endif()
 
-add_subdirectory(pybind11)
+find_package(pybind11 CONFIG REQUIRED)
 
-pybind11_add_module(python MODULE EXCLUDE_FROM_ALL SYSTEM THIN_LTO)
+pybind11_add_module(python MODULE EXCLUDE_FROM_ALL THIN_LTO)
 
 target_link_libraries(python
   PRIVATE

data/vendor/datasketches-cpp/python/README.md

@@ -1,76 +1,57 @@
-# Python Wrapper for Apache DataSketches
+<img src="https://raw.githubusercontent.com/apache/datasketches-website/master/logos/svg/datasketches-HorizontalColor-TM.svg" width="75%" alt="Apache DataSketchs Logo">
 
-## Installation
+# The Apache DataSketches Library for Python
 
-The release files do not include the needed python binding library ([pybind11](https://github.com/pybind/pybind11)). If building
-from a relase package, you must ensure that the pybind11 directory points to a local copy of pybind11.
+This is the official version of the [Apache DataSketches](https://datasketches.apache.org) Python library.
 
-An official pypi build is eventually planned but not yet available.
+In the analysis of big data there are often problem queries that don’t scale because they require huge compute resources and time to generate exact results. Examples include count distinct, quantiles, most-frequent items, joins, matrix computations, and graph analysis.
 
-If you instead want to take a (possibly ill-advised) gamble on the current state of the master branch being useable, you can run:
-```pip install git+https://github.com/apache/datasketches-cpp.git```
+If approximate results are acceptable, there is a class of specialized algorithms, called streaming algorithms, or sketches that can produce results orders-of magnitude faster and with mathematically proven error bounds. For interactive queries there may not be other viable alternatives, and in the case of real-time analysis, sketches are the only known solution.
 
-## Developer Instructions
-
-### Building
-
-When cloning the source repository, you should include the pybind11 submodule with the `--recursive` option to the clone command:
-```
-git clone --recursive https://github.com/apache/datasketches-cpp.git
-cd datasketches-cpp
-python -m pip install --upgrade pip setuptools wheel numpy
-python setup.py build
-```
+This package provides a variety of sketches as described below. Wherever a specific type of sketch exists in Apache DataSketches packages for other languages, the sketches will be portable between languages (for platforms with the same endianness).
 
-If you cloned without `--recursive`, you can add the submodule post-checkout using `git submodule update --init --recursive`.
+## Building and Installation
 
-### Installing
+Once cloned, the library can be installed by running `python -m pip install .` in the project root directory, which will also install the necessary dependencies, namely numpy and [pybind11[global]](https://github.com/pybind/pybind11).
 
-Assuming you have already checked out the library and any dependent submodules, install by simply replacing the lsat
-line of the build command with `python setup.py install`.
+If you prefer to call the `setup.py` build script directly, you must first install `pybind11[global]`, as well as any other dependencies listed under the build-system section in `pyproject.toml`.
 
-### Unit tests
-
-The python tests are run with `tox`. To ensure you have all the needed packages, from the package base directory run:
-```
-python -m pip install --upgrade pip setuptools wheel numpy tox
-tox
-```
+The library is also available from PyPI via `python -m pip install datasketches`.
 
 ## Usage
 
-Having installed the library, loading the Apache Datasketches library in Python is simple: `import datasketches`.
+Having installed the library, loading the Apache Datasketches Library in Python is simple: `import datasketches`.
 
 ## Available Sketch Classes
 
 - KLL (Absolute Error Quantiles)
-    - `kll_ints_sketch`
-    - `kll_floats_sketch`
+  - `kll_ints_sketch`
+  - `kll_floats_sketch`
 - REQ (Relative Error Quantiles)
-    - `req_ints_sketch`
-    - `req_floats_sketch`
+  - `req_ints_sketch`
+  - `req_floats_sketch`
 - Frequent Items
-    - `frequent_strings_sketch`
-    - Error types are `frequent_items_error_type.{NO_FALSE_NEGATIVES | NO_FALSE_POSITIVES}`
+  - `frequent_strings_sketch`
+  - Error types are `frequent_items_error_type.{NO_FALSE_NEGATIVES | NO_FALSE_POSITIVES}`
 - Theta
-    - `update_theta_sketch`
-    - `compact_theta_sketch` (cannot be instantiated directly)
-    - `theta_union`
-    - `theta_intersection`
-    - `theta_a_not_b`
+  - `update_theta_sketch`
+  - `compact_theta_sketch` (cannot be instantiated directly)
+  - `theta_union`
+  - `theta_intersection`
+  - `theta_a_not_b`
 - HLL
-    - `hll_sketch`
-    - `hll_union`
-    - Target HLL types are `tgt_hll_type.{HLL_4 | HLL_6 | HLL_8}`
+  - `hll_sketch`
+  - `hll_union`
+  - Target HLL types are `tgt_hll_type.{HLL_4 | HLL_6 | HLL_8}`
 - CPC
-    - `cpc_sketch`
-    - `cpc_union`
+  - `cpc_sketch`
+  - `cpc_union`
 - VarOpt Sampling
-    - `var_opt_sketch`
-    - `var_opt_union`
+  - `var_opt_sketch`
+  - `var_opt_union`
 - Vector of KLL
-    - `vector_of_kll_ints_sketches`
-    - `vector_of_kll_floats_sketches`
+  - `vector_of_kll_ints_sketches`
+  - `vector_of_kll_floats_sketches`
 
 ## Known Differences from C++
 
@@ -79,3 +60,22 @@ The Python API largely mirrors the C++ API, with a few minor exceptions: The pri
 The Vector of KLL object is currently exclusive to python, and holds an array of independent KLL sketches. This is useful for creating a set of KLL sketches over a vector and has been designed to allow input as either a vector or a matrix of multiple vectors.
 
 We have also removed reliance on a builder class for theta sketches as Python allows named arguments to the constructor, not strictly positional arguments.
+
+## Developer Instructions
+
+The only developer-specific instructions relate to running unit tests.
+
+### Unit tests
+
+The Python unit tests are run with `tox`. To ensure you have all the needed package, from the package base directory run:
+
+```bash
+python -m pip install --upgrade tox
+tox
+```
+
+## License
+
+The Apache DataSketches Library is distrubted under an Apache 2.0 License.
+
+There may be precompiled binaries provided as a convenience and distributed through PyPI via [https://pypi.org/project/datasketches/] contain compiled code from [pybind11](https://github.com/pybind/pybind11), which is distributed under a BSD license.

data/vendor/datasketches-cpp/python/pybind11Path.cmd

@@ -0,0 +1,3 @@
+@echo off
+:: Takes path to the Python interpreter and returns the path to pybind11
+%1 -m pip show pybind11 | %1 -c "import sys,re;[sys.stdout.write(re.sub('^Location:\\s+','',line)) for line in sys.stdin if re.search('^Location:\\s+',line)]"

data/vendor/datasketches-cpp/python/src/cpc_wrapper.cpp

@@ -53,7 +53,7 @@ void init_cpc(py::module &m) {
   using namespace datasketches;
 
   py::class_<cpc_sketch>(m, "cpc_sketch")
-    .def(py::init<uint8_t, uint64_t>(), py::arg("lg_k")=CPC_DEFAULT_LG_K, py::arg("seed")=DEFAULT_SEED)
+    .def(py::init<uint8_t, uint64_t>(), py::arg("lg_k")=cpc_constants::DEFAULT_LG_K, py::arg("seed")=DEFAULT_SEED)
     .def(py::init<const cpc_sketch&>())
     .def("__str__", &cpc_sketch::to_string,
          "Produces a string summary of the sketch")

data/vendor/datasketches-cpp/python/src/kll_wrapper.cpp

@@ -116,7 +116,7 @@ void bind_kll_sketch(py::module &m, const char* name) {
   using namespace datasketches;
 
   py::class_<kll_sketch<T>>(m, name)
-    .def(py::init<uint16_t>(), py::arg("k")=kll_sketch<T>::DEFAULT_K)
+    .def(py::init<uint16_t>(), py::arg("k")=kll_constants::DEFAULT_K)
     .def(py::init<const kll_sketch<T>&>())
     .def("update", (void (kll_sketch<T>::*)(const T&)) &kll_sketch<T>::update, py::arg("item"),
          "Updates the sketch with the given value")

data/vendor/datasketches-cpp/python/src/theta_wrapper.cpp

@@ -103,7 +103,7 @@ void init_theta(py::module &m) {
 
   py::class_<update_theta_sketch, theta_sketch>(m, "update_theta_sketch")
     .def(py::init(&dspy::update_theta_sketch_factory),
-         py::arg("lg_k")=update_theta_sketch::builder::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED)
+         py::arg("lg_k")=theta_constants::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED)
     .def(py::init<const update_theta_sketch&>())
     .def("update", (void (update_theta_sketch::*)(int64_t)) &update_theta_sketch::update, py::arg("datum"),
          "Updates the sketch with the given integral value")
@@ -127,7 +127,7 @@ void init_theta(py::module &m) {
 
   py::class_<theta_union>(m, "theta_union")
     .def(py::init(&dspy::theta_union_factory),
-         py::arg("lg_k")=update_theta_sketch::builder::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED)
+         py::arg("lg_k")=theta_constants::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED)
     .def("update", &theta_union::update<const theta_sketch&>, py::arg("sketch"),
          "Updates the union with the given sketch")
     .def("get_result", &theta_union::get_result, py::arg("ordered")=true,

data/vendor/datasketches-cpp/python/src/vector_of_kll.cpp

@@ -29,14 +29,20 @@ namespace py = pybind11;
 
 namespace datasketches {
 
+namespace vector_of_kll_constants {
+  static const uint32_t DEFAULT_K = kll_constants::DEFAULT_K;
+  static const uint32_t DEFAULT_D = 1;
+}
+
 // Wrapper class for Numpy compatibility
 template <typename T, typename C = std::less<T>, typename S = serde<T>>
 class vector_of_kll_sketches {
   public:
-    static const uint32_t DEFAULT_K = kll_sketch<T, C, S>::DEFAULT_K;
-    static const uint32_t DEFAULT_D = 1;
+    // TODO: Redundant and deprecated. Will be removed in next major version release.
+    static const uint32_t DEFAULT_K = vector_of_kll_constants::DEFAULT_K;
+    static const uint32_t DEFAULT_D = vector_of_kll_constants::DEFAULT_D;
 
-    explicit vector_of_kll_sketches(uint32_t k = DEFAULT_K, uint32_t d = DEFAULT_D);
+    explicit vector_of_kll_sketches(uint32_t k = vector_of_kll_constants::DEFAULT_K, uint32_t d = vector_of_kll_constants::DEFAULT_D);
     vector_of_kll_sketches(const vector_of_kll_sketches& other);
     vector_of_kll_sketches(vector_of_kll_sketches&& other) noexcept;
     vector_of_kll_sketches<T,C,S>& operator=(const vector_of_kll_sketches& other);
@@ -432,8 +438,8 @@ void bind_vector_of_kll_sketches(py::module &m, const char* name) {
   using namespace datasketches;
 
   py::class_<vector_of_kll_sketches<T>>(m, name)
-    .def(py::init<uint32_t, uint32_t>(), py::arg("k")=vector_of_kll_sketches<T>::DEFAULT_K, 
-                                         py::arg("d")=vector_of_kll_sketches<T>::DEFAULT_D)
+    .def(py::init<uint32_t, uint32_t>(), py::arg("k")=vector_of_kll_constants::DEFAULT_K, 
+                                         py::arg("d")=vector_of_kll_constants::DEFAULT_D)
     .def(py::init<const vector_of_kll_sketches<T>&>())
     // allow user to retrieve k or d, in case it's instantiated w/ defaults
     .def("get_k", &vector_of_kll_sketches<T>::get_k,

data/vendor/datasketches-cpp/python/tests/kll_test.py

@@ -30,10 +30,10 @@ class KllTest(unittest.TestCase):
       kll.update(0.0)
 
       # 0 should be near the median
-      self.assertAlmostEqual(0.5, kll.get_rank(0.0), delta=0.025)
+      self.assertAlmostEqual(0.5, kll.get_rank(0.0), delta=0.035)
       
       # the median should be near 0
-      self.assertAlmostEqual(0.0, kll.get_quantile(0.5), delta=0.025)
+      self.assertAlmostEqual(0.0, kll.get_quantile(0.5), delta=0.035)
 
       # we also track the min/max independently from the rest of the data
       # which lets us know the full observed data range

data/vendor/datasketches-cpp/python/tests/req_test.py

@@ -30,10 +30,10 @@ class reqTest(unittest.TestCase):
       req.update(0.0)
 
       # 0 should be near the median
-      self.assertAlmostEqual(0.5, req.get_rank(0.0), delta=0.03)
+      self.assertAlmostEqual(0.5, req.get_rank(0.0), delta=0.045)
       
       # the median should be near 0
-      self.assertAlmostEqual(0.0, req.get_quantile(0.5), delta=0.03)
+      self.assertAlmostEqual(0.0, req.get_quantile(0.5), delta=0.045)
 
       # we also track the min/max independently from the rest of the data
       # which lets us know the full observed data range

data/vendor/datasketches-cpp/python/tests/vector_of_kll_test.py

@@ -39,9 +39,9 @@ class VectorOfKllSketchesTest(unittest.TestCase):
         kll.update(dat)
 
       # 0 should be near the median
-      np.testing.assert_allclose(0.5, kll.get_ranks(0.0), atol=0.025)
+      np.testing.assert_allclose(0.5, kll.get_ranks(0.0), atol=0.035)
       # the median should be near 0
-      np.testing.assert_allclose(0.0, kll.get_quantiles(0.5), atol=0.025)
+      np.testing.assert_allclose(0.0, kll.get_quantiles(0.5), atol=0.035)
       # we also track the min/max independently from the rest of the data
       # which lets us know the full observed data range
       np.testing.assert_allclose(kll.get_min_values(), smin)
@@ -118,9 +118,9 @@ class VectorOfKllSketchesTest(unittest.TestCase):
         kll.update(dat)
 
       # 0 should be near the median
-      np.testing.assert_allclose(0.5, kll.get_ranks(0.0), atol=0.025)
+      np.testing.assert_allclose(0.5, kll.get_ranks(0.0), atol=0.035)
       # the median should be near 0
-      np.testing.assert_allclose(0.0, kll.get_quantiles(0.5), atol=0.025)
+      np.testing.assert_allclose(0.0, kll.get_quantiles(0.5), atol=0.035)
       # we also track the min/max independently from the rest of the data
       # which lets us know the full observed data range
       np.testing.assert_allclose(kll.get_min_values(), smin)

data/vendor/datasketches-cpp/sampling/include/var_opt_sketch.hpp

@@ -51,18 +51,23 @@ struct subset_summary {
   double total_sketch_weight;
 };
 
-enum resize_factor { X1 = 0, X2, X4, X8 };
-
 template <typename T, typename S, typename A> class var_opt_union; // forward declaration
 
+namespace var_opt_constants {
+    const resize_factor DEFAULT_RESIZE_FACTOR = resize_factor::X8;
+    const uint32_t MAX_K = ((uint32_t) 1 << 31) - 2; 
+}
+
 template <typename T, typename S = serde<T>, typename A = std::allocator<T>>
 class var_opt_sketch {
 
   public:
-    static const resize_factor DEFAULT_RESIZE_FACTOR = X8;
-    static const uint32_t MAX_K = ((uint32_t) 1 << 31) - 2;
+    static const resize_factor DEFAULT_RESIZE_FACTOR = var_opt_constants::DEFAULT_RESIZE_FACTOR;
+    static const uint32_t MAX_K = var_opt_constants::MAX_K;
 
-    explicit var_opt_sketch(uint32_t k, resize_factor rf = DEFAULT_RESIZE_FACTOR, const A& allocator = A());
+    explicit var_opt_sketch(uint32_t k,
+      resize_factor rf = var_opt_constants::DEFAULT_RESIZE_FACTOR,
+      const A& allocator = A());
     var_opt_sketch(const var_opt_sketch& other);
     var_opt_sketch(var_opt_sketch&& other) noexcept;
 

data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp

@@ -128,7 +128,7 @@ var_opt_sketch<T,S,A>::var_opt_sketch(T* data, double* weights, size_t len,
   r_(r_count),
   n_(n),
   total_wt_r_(total_wt_r),
-  rf_(DEFAULT_RESIZE_FACTOR),
+  rf_(var_opt_constants::DEFAULT_RESIZE_FACTOR),
   curr_items_alloc_(len),
   filled_data_(n > k),
   allocator_(allocator),

data/vendor/datasketches-cpp/setup.py

@@ -49,8 +49,9 @@ class CMakeBuild(build_ext):
             os.path.dirname(self.get_ext_fullpath(ext.name)))
         cmake_args =  ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir]
         cmake_args += ['-DWITH_PYTHON=True']
+        cmake_args += ['-DCMAKE_CXX_STANDARD=11']
         # ensure we use a consistent python version
-        cmake_args += ['-DPYTHON_EXECUTABLE=' + sys.executable]
+        cmake_args += ['-DPython3_EXECUTABLE=' + sys.executable]
         cfg = 'Debug' if self.debug else 'Release'
         build_args = ['--config', cfg]
 
@@ -59,7 +60,8 @@ class CMakeBuild(build_ext):
                            cfg.upper(),
                            extdir)]
             if sys.maxsize > 2**32:
-                cmake_args += ['-A', 'x64']
+                cmake_args += ['-T', 'host=x64']
+                cmake_args += ['-DCMAKE_GENERATOR_PLATFORM=x64']
                 build_args += ['--', '/m']
         else:
             cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
@@ -74,23 +76,24 @@ class CMakeBuild(build_ext):
         subprocess.check_call(['cmake', ext.sourcedir] + cmake_args,
                               cwd=self.build_temp, env=env)
         subprocess.check_call(['cmake', '--build', '.', '--target', 'python'] + build_args,
-                              cwd=self.build_temp)
+                              cwd=self.build_temp, env=env)
         print() # add an empty line to pretty print
 
 setup(
     name='datasketches',
-    version='3.1.0',
-    author='Apache DataSketches Developers',
+    version='3.2.0.1',
+    author='Apache Software Foundation',
     author_email='dev@datasketches.apache.org',
-    description='A wrapper for the C++ Apache DataSketches library',
+    description='The Apache DataSketches Library for Python',
     license='Apache License 2.0',
     url='http://datasketches.apache.org',
     long_description=open('python/README.md').read(),
+    long_description_content_type='text/markdown',
     packages=find_packages('python'), # python pacakges only in this dir
     package_dir={'':'python'},
     # may need to add all source paths for sdist packages w/o MANIFEST.in
     ext_modules=[CMakeExtension('datasketches')],
     cmdclass={'build_ext': CMakeBuild},
-    setup_requires=['setuptools_scm','tox-setuptools'],
+    install_requires=['numpy'],
     zip_safe=False
 )

data/vendor/datasketches-cpp/theta/include/theta_constants.hpp

@@ -21,14 +21,19 @@
 #define THETA_CONSTANTS_HPP_
 
 #include <climits>
+#include "common_defs.hpp"
 
 namespace datasketches {
 
 namespace theta_constants {
-  enum resize_factor { X1, X2, X4, X8 };
-  static const uint64_t MAX_THETA = LLONG_MAX; // signed max for compatibility with Java
-  static const uint8_t MIN_LG_K = 5;
-  static const uint8_t MAX_LG_K = 26;
+  using resize_factor = datasketches::resize_factor;
+  //enum resize_factor { X1, X2, X4, X8 };
+  const uint64_t MAX_THETA = LLONG_MAX; // signed max for compatibility with Java
+  const uint8_t MIN_LG_K = 5;
+  const uint8_t MAX_LG_K = 26;
+
+  const uint8_t DEFAULT_LG_K = 12;
+  const resize_factor DEFAULT_RESIZE_FACTOR = resize_factor::X8;
 }
 
 } /* namespace datasketches */

data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base.hpp

@@ -94,11 +94,14 @@ struct theta_update_sketch_base {
 template<typename Derived, typename Allocator>
 class theta_base_builder {
 public:
+  // TODO: Redundant and deprecated. Will be removed in next major verison release.
   using resize_factor = theta_constants::resize_factor;
   static const uint8_t MIN_LG_K = theta_constants::MIN_LG_K;
   static const uint8_t MAX_LG_K = theta_constants::MAX_LG_K;
-  static const uint8_t DEFAULT_LG_K = 12;
-  static const resize_factor DEFAULT_RESIZE_FACTOR = resize_factor::X8;
+  // TODO: The following defaults are redundant and deprecated. Will be removed in the
+  //       next major version release
+  static const uint8_t DEFAULT_LG_K = theta_constants::DEFAULT_LG_K;
+  static const resize_factor DEFAULT_RESIZE_FACTOR = theta_constants::DEFAULT_RESIZE_FACTOR;
 
   /**
    * Creates and instance of the builder with default parameters.

data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp

@@ -271,7 +271,11 @@ void theta_update_sketch_base<EN, EK, A>::consolidate_non_empty(EN* entries, siz
 
 template<typename Derived, typename Allocator>
 theta_base_builder<Derived, Allocator>::theta_base_builder(const Allocator& allocator):
-allocator_(allocator), lg_k_(DEFAULT_LG_K), rf_(DEFAULT_RESIZE_FACTOR), p_(1), seed_(DEFAULT_SEED) {}
+allocator_(allocator),
+lg_k_(theta_constants::DEFAULT_LG_K),
+rf_(theta_constants::DEFAULT_RESIZE_FACTOR),
+p_(1),
+seed_(DEFAULT_SEED) {}
 
 template<typename Derived, typename Allocator>
 Derived& theta_base_builder<Derived, Allocator>::set_lg_k(uint8_t lg_k) {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datasketches
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-17 00:00:00.000000000 Z
+date: 2021-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -177,6 +177,7 @@ files:
 - vendor/datasketches-cpp/python/jupyter/HLLSketch.ipynb
 - vendor/datasketches-cpp/python/jupyter/KLLSketch.ipynb
 - vendor/datasketches-cpp/python/jupyter/ThetaSketchNotebook.ipynb
+- vendor/datasketches-cpp/python/pybind11Path.cmd
 - vendor/datasketches-cpp/python/src/__init__.py
 - vendor/datasketches-cpp/python/src/cpc_wrapper.cpp
 - vendor/datasketches-cpp/python/src/datasketches.cpp