datasketches 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9131edb6c019db8cd0dbb98aaf1321ba213efe5911ea17dd37f4e2a6cd8e7125
-  data.tar.gz: dc60612514895814bb7e920d8e66029b10abb820a6a6617e6426cc725482d6e2
+  metadata.gz: d80465f08285b46a56497ab21c0b77afa47eb183f49d463548113480a86a0128
+  data.tar.gz: 0e88f707d65bb9b40790c6e9a3a378395cddcc5ebd4ff367fc2f31523b6efc77
 SHA512:
-  metadata.gz: b030673f22e3c02c7a1805a9c8378e305b4878c18c4c92c3cdccaa54ea2ea9a87871b6a858563a02be510f6371bfe1c34e76386ccad975da05a6bd09071c5ee9
-  data.tar.gz: 11b1cbd76b5e47547b54ef39c195a2d9caa2a1a2a7ec3b8ed33be87b0e98adfa6cbef1ac8e6f5af6d4b57390731b10949f13620b4f916e15b7c73a449dfdfaa5
+  metadata.gz: 7b19e71cfeccb68714641f6a0cf84e24939f658dc52d328bd5cb05af8433e9622cfe4e4b51a1ed90929be273b1024989261cceae0447a7a328734a1e1c239509
+  data.tar.gz: ba51c3c7512c91bf77f6a309a28e4b2e7914812faa4bef78362eeecdc06ebcf4faaaf80931f63688acfe3ee5e27ebd1d325c9848ad3f14fc96181ea6cbf1b909

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.3 (2024-08-02)
+
+- Updated DataSketches to 5.1.0
+
 ## 0.4.2 (2024-01-13)
 
 - Updated DataSketches to 5.0.2

data/NOTICE

@@ -1,5 +1,5 @@
 Apache DataSketches C++ and Python
-Copyright 2023 The Apache Software Foundation
+Copyright 2024 The Apache Software Foundation
 
 Copyright 2015-2018 Yahoo Inc.
 Copyright 2019-2020 Verizon Media

data/README.md

@@ -2,7 +2,7 @@
 
 [DataSketches](https://datasketches.apache.org/) - sketch data structures - for Ruby
 
-[![Build Status](https://github.com/ankane/datasketches-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/datasketches-ruby/actions)
+[![Build Status](https://github.com/ankane/datasketches-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/datasketches-ruby/actions)
 
 ## Installation
 

data/lib/datasketches/version.rb

@@ -1,3 +1,3 @@
 module DataSketches
-  VERSION = "0.4.2"
+  VERSION = "0.4.3"
 end

data/vendor/datasketches-cpp/CMakeLists.txt

@@ -118,6 +118,7 @@ add_subdirectory(req)
 add_subdirectory(quantiles)
 add_subdirectory(count)
 add_subdirectory(density)
+add_subdirectory(tdigest)
 
 if (WITH_PYTHON)
   add_subdirectory(python)

data/vendor/datasketches-cpp/NOTICE

@@ -1,5 +1,5 @@
-Apache DataSketches C++ and Python
-Copyright 2023 The Apache Software Foundation
+Apache DataSketches C++
+Copyright 2024 The Apache Software Foundation
 
 Copyright 2015-2018 Yahoo Inc.
 Copyright 2019-2020 Verizon Media

data/vendor/datasketches-cpp/README.md

@@ -3,8 +3,7 @@ This is the core C++ component of the Apache DataSketches library.  It contains
 
 This component is also a dependency of other components of the library that create adaptors for target systems, such as PostgreSQL.
 
-Note that we have a parallel core component for Java implementations of the same sketch algorithms, 
-[datasketches-java](https://github.com/apache/datasketches-java).
+Note that we have a parallel core component for [Java]((https://github.com/apache/datasketches-java) and [Python]((https://github.com/apache/datasketches-python) implementations of the same sketch algorithms.
 
 Please visit the main [Apache DataSketches website](https://datasketches.apache.org) for more information. 
 
@@ -104,4 +103,4 @@ from GitHub using CMake's `ExternalProject` module. The code would look somethin
     target_include_directories(my_dependent_target 
                                 PRIVATE ${datasketches_INSTALL_DIR}/include/DataSketches)
     add_dependencies(my_dependent_target datasketches)
-```
+```

data/vendor/datasketches-cpp/common/CMakeLists.txt

@@ -29,8 +29,6 @@ target_include_directories(common
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
 )
 
-target_compile_features(common INTERFACE cxx_std_11)
-
 install(TARGETS common EXPORT ${PROJECT_NAME})
 
 install(FILES 

data/vendor/datasketches-cpp/common/include/MurmurHash3.h

@@ -71,10 +71,10 @@ typedef struct {
 // Block read - if your platform needs to do endian-swapping or can only
 // handle aligned reads, do the conversion here
 
-MURMUR3_FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, size_t i )
+MURMUR3_FORCE_INLINE uint64_t getblock64 ( const uint8_t * p, size_t i )
 {
   uint64_t res;
-  memcpy(&res, p + i, sizeof(res));
+  memcpy(&res, p + i * sizeof(uint64_t), sizeof(res));
   return res;
 }
 
@@ -104,13 +104,12 @@ MURMUR3_FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes,
 
   // Number of full 128-bit blocks of 16 bytes.
   // Possible exclusion of a remainder of up to 15 bytes.
-  const size_t nblocks = lenBytes >> 4; // bytes / 16 
+  const size_t nblocks = lenBytes >> 4; // bytes / 16
 
   // Process the 128-bit blocks (the body) into the hash
-  const uint64_t* blocks = (const uint64_t*)(data);
   for (size_t i = 0; i < nblocks; ++i) { // 16 bytes per block
-    uint64_t k1 = getblock64(blocks, i * 2 + 0);
-    uint64_t k2 = getblock64(blocks, i * 2 + 1);
+    uint64_t k1 = getblock64(data, i * 2 + 0);
+    uint64_t k2 = getblock64(data, i * 2 + 1);
 
     k1 *= c1; k1  = MURMUR3_ROTL64(k1,31); k1 *= c2; out.h1 ^= k1;
     out.h1 = MURMUR3_ROTL64(out.h1,27);

data/vendor/datasketches-cpp/common/include/common_defs.hpp

@@ -91,6 +91,23 @@ static inline void write(std::ostream& os, const T* ptr, size_t size_bytes) {
   os.write(reinterpret_cast<const char*>(ptr), size_bytes);
 }
 
+template<typename T>
+T byteswap(T value) {
+  char* ptr = static_cast<char*>(static_cast<void*>(&value));
+  const int len = sizeof(T);
+  for (size_t i = 0; i < len / 2; ++i) {
+    std::swap(ptr[i], ptr[len - i - 1]);
+  }
+  return value;
+}
+
+template<typename T>
+static inline T read_big_endian(std::istream& is) {
+  T value;
+  is.read(reinterpret_cast<char*>(&value), sizeof(T));
+  return byteswap(value);
+}
+
 // wrapper for iterators to implement operator-> returning temporary value
 template<typename T>
 class return_value_holder {

data/vendor/datasketches-cpp/count/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(count
         )
 
 target_link_libraries(count INTERFACE common)
-target_compile_features(count INTERFACE cxx_std_11)
 
 install(TARGETS count
         EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/cpc/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(cpc
 )
 
 target_link_libraries(cpc INTERFACE common)
-target_compile_features(cpc INTERFACE cxx_std_11)
 
 install(TARGETS cpc
   EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/cpc/include/cpc_compressor.hpp

@@ -44,6 +44,10 @@ template<typename A> class cpc_compressor;
 template<typename A>
 inline cpc_compressor<A>& get_compressor();
 
+// function called atexit to clean up compression tables
+template<typename A>
+void destroy_compressor();
+
 template<typename A>
 class cpc_compressor {
 public:
@@ -109,8 +113,10 @@ private:
   };
 
   cpc_compressor();
-  template<typename T> friend cpc_compressor<T>& get_compressor();
+  friend cpc_compressor& get_compressor<A>();
+  
   ~cpc_compressor();
+  friend void destroy_compressor<A>();
 
   void make_decoding_tables(); // call this at startup
   void free_decoding_tables(); // call this at the end

data/vendor/datasketches-cpp/cpc/include/cpc_compressor_impl.hpp

@@ -22,9 +22,11 @@
 #ifndef CPC_COMPRESSOR_IMPL_HPP_
 #define CPC_COMPRESSOR_IMPL_HPP_
 
+#include <cstdlib>
 #include <memory>
 #include <stdexcept>
 
+#include "common_defs.hpp"
 #include "compression_data.hpp"
 #include "cpc_util.hpp"
 #include "cpc_common.hpp"
@@ -36,9 +38,17 @@ namespace datasketches {
 template<typename A>
 cpc_compressor<A>& get_compressor() {
   static cpc_compressor<A>* instance = new cpc_compressor<A>(); // use new for global initialization
+  static int reg_result = std::atexit(destroy_compressor<A>); // just to clean up a little more nicely; don't worry if it fails
+  unused(reg_result);
   return *instance;
 }
 
+// register to call compressor destructor at exit
+template<typename A>
+void destroy_compressor() {
+  delete std::addressof(get_compressor<A>());
+}
+
 template<typename A>
 cpc_compressor<A>::cpc_compressor() {
   make_decoding_tables();

data/vendor/datasketches-cpp/density/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(density
 )
 
 target_link_libraries(density INTERFACE common)
-target_compile_features(density INTERFACE cxx_std_11)
 
 install(TARGETS density
   EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/fi/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(fi
 )
 
 target_link_libraries(fi INTERFACE common)
-target_compile_features(fi INTERFACE cxx_std_11)
 
 install(TARGETS fi
   EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/hll/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(hll
 )
 
 target_link_libraries(hll INTERFACE common)
-target_compile_features(hll INTERFACE cxx_std_11)
 
 install(TARGETS hll
   EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/kll/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(kll
 )
 
 target_link_libraries(kll INTERFACE common)
-target_compile_features(kll INTERFACE cxx_std_11)
 
 install(TARGETS kll
   EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/kll/test/kll_sketch_custom_type_test.cpp

@@ -31,11 +31,14 @@ using alloc = test_allocator<test_type>;
 
 TEST_CASE("kll sketch custom type", "[kll_sketch]") {
 
-  // setup section
   test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
 
   SECTION("compact level zero") {
     kll_test_type_sketch sketch(8, test_type_less(), 0);
+    REQUIRE(test_allocator_total_bytes != 0);
+    REQUIRE(test_allocator_net_allocations != 0);
+
     REQUIRE_THROWS_AS(sketch.get_quantile(0), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_min_item(), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_max_item(), std::runtime_error);
@@ -146,10 +149,8 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
     REQUIRE(sketch2.get_n() == 11);
   }
 
-  // cleanup
-  if (test_allocator_total_bytes != 0) {
-    REQUIRE(test_allocator_total_bytes == 0);
-  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
 }
 
 } /* namespace datasketches */

data/vendor/datasketches-cpp/quantiles/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(quantiles
 )
 
 target_link_libraries(quantiles INTERFACE common)
-target_compile_features(quantiles INTERFACE cxx_std_11)
 
 install(TARGETS quantiles
   EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/req/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(req
 )
 
 target_link_libraries(req INTERFACE common)
-target_compile_features(req INTERFACE cxx_std_11)
 
 install(TARGETS req
   EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/sampling/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(sampling
 )
 
 target_link_libraries(sampling INTERFACE common)
-target_compile_features(sampling INTERFACE cxx_std_11)
 
 install(TARGETS sampling
   EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/sampling/include/ebpps_sample.hpp

@@ -37,14 +37,14 @@ class ebpps_sample {
   public:
     explicit ebpps_sample(uint32_t k, const A& allocator = A());
 
-    // constructor used to create a sample to merge one itme
-    template<typename TT>
-    ebpps_sample(TT&& item, double theta, const A& allocator = A());
-
     // for deserialization
     class items_deleter;
     ebpps_sample(std::vector<T, A>&& data, optional<T>&& partial_item, double c, const A& allocator = A());
 
+    // used instead of having a single-item constructor for update/merge calls
+    template<typename TT>
+    void replace_content(TT&& item, double theta);
+
     void reset();
     void downsample(double theta);
 

data/vendor/datasketches-cpp/sampling/include/ebpps_sample_impl.hpp

@@ -41,22 +41,6 @@ ebpps_sample<T,A>::ebpps_sample(uint32_t reserved_size, const A& allocator) :
     data_.reserve(reserved_size);
   }
 
-template<typename T, typename A>
-template<typename TT>
-ebpps_sample<T,A>::ebpps_sample(TT&& item, double theta, const A& allocator) :
-  allocator_(allocator),
-  c_(theta),
-  partial_item_(),
-  data_(allocator)
-  {
-    if (theta == 1.0) {
-      data_.reserve(1);
-      data_.emplace_back(std::forward<TT>(item));
-    } else {
-      partial_item_.emplace(std::forward<TT>(item));
-    }
-  }
-
 template<typename T, typename A>
 ebpps_sample<T,A>::ebpps_sample(std::vector<T, A>&& data, optional<T>&& partial_item, double c, const A& allocator) :
   allocator_(allocator),
@@ -65,6 +49,19 @@ ebpps_sample<T,A>::ebpps_sample(std::vector<T, A>&& data, optional<T>&& partial_
   data_(data, allocator)
   {}
 
+template<typename T, typename A>
+template<typename TT>
+void ebpps_sample<T,A>::replace_content(TT&& item, double theta) {
+  c_ = theta;
+  data_.clear();
+  partial_item_.reset();
+  if (theta == 1.0) {
+    data_.emplace_back(std::forward<TT>(item));
+  } else {
+    partial_item_.emplace(std::forward<TT>(item));
+  }
+}
+
 template<typename T, typename A>
 auto ebpps_sample<T,A>::get_sample() const -> result_type {
   double unused;

data/vendor/datasketches-cpp/sampling/include/ebpps_sketch.hpp

@@ -43,7 +43,7 @@ namespace ebpps_constants {
  * From: "Exact PPS Sampling with Bounded Sample Size",
  * B. Hentschel, P. J. Haas, Y. Tian. Information Processing Letters, 2023.
  * 
- * This sketch samples data from a stream of items propotional to the weight of each item.
+ * This sketch samples data from a stream of items proportional to the weight of each item.
  * The sample guarantees the presence of an item in the result is proportional to that item's
  * portion of the total weight seen by the sketch, and returns a sample no larger than size k.
  * 
@@ -256,6 +256,8 @@ class ebpps_sketch {
 
     ebpps_sample<T,A> sample_;      // Object holding the current state of the sample
 
+    ebpps_sample<T,A> tmp_;         // Temporary sample of size 1 used in updates
+
     // handles merge after ensuring other.cumulative_wt_ <= this->cumulative_wt_
     // so we can send items in individually
     template<typename O>

data/vendor/datasketches-cpp/sampling/include/ebpps_sketch_impl.hpp

@@ -40,7 +40,8 @@ ebpps_sketch<T, A>::ebpps_sketch(uint32_t k, const A& allocator) :
   cumulative_wt_(0.0),
   wt_max_(0.0),
   rho_(1.0),
-  sample_(check_k(k), allocator)
+  sample_(check_k(k), allocator),
+  tmp_(1, allocator)
   {}
 
 template<typename T, typename A>
@@ -53,7 +54,8 @@ ebpps_sketch<T,A>::ebpps_sketch(uint32_t k, uint64_t n, double cumulative_wt,
   cumulative_wt_(cumulative_wt),
   wt_max_(wt_max),
   rho_(rho),
-  sample_(sample)
+  sample_(sample),
+  tmp_(1, allocator)
   {}
 
 template<typename T, typename A>
@@ -148,9 +150,8 @@ void ebpps_sketch<T, A>::internal_update(FwdItem&& item, double weight) {
   if (cumulative_wt_ > 0.0)
     sample_.downsample(new_rho / rho_);
   
-  ebpps_sample<T,A> tmp(conditional_forward<FwdItem>(item), new_rho * weight, allocator_);
-
-  sample_.merge(tmp);
+  tmp_.replace_content(conditional_forward<FwdItem>(item), new_rho * weight);
+  sample_.merge(tmp_);
 
   cumulative_wt_ = new_cum_wt;
   wt_max_ = new_wt_max;
@@ -240,9 +241,8 @@ void ebpps_sketch<T, A>::internal_merge(O&& sk) {
     if (cumulative_wt_ > 0.0)
       sample_.downsample(new_rho / rho_);
   
-    ebpps_sample<T,A> tmp(conditional_forward<O>(items[i]), new_rho * avg_wt, allocator_);
-
-    sample_.merge(tmp);
+    tmp_.replace_content(conditional_forward<O>(items[i]), new_rho * avg_wt);
+    sample_.merge(tmp_);
 
     cumulative_wt_ = new_cum_wt;
     rho_ = new_rho;
@@ -259,9 +259,8 @@ void ebpps_sketch<T, A>::internal_merge(O&& sk) {
     if (cumulative_wt_ > 0.0)
       sample_.downsample(new_rho / rho_);
   
-    ebpps_sample<T,A> tmp(conditional_forward<O>(other_sample.get_partial_item()), new_rho * other_c_frac * avg_wt, allocator_);
-
-    sample_.merge(tmp);
+    tmp_.replace_content(conditional_forward<O>(other_sample.get_partial_item()), new_rho * other_c_frac * avg_wt);
+    sample_.merge(tmp_);
 
     cumulative_wt_ = new_cum_wt;
     rho_ = new_rho;

data/vendor/datasketches-cpp/sampling/test/ebpps_sample_test.cpp

@@ -42,14 +42,15 @@ TEST_CASE("ebpps sample: basic initialization", "[ebpps_sketch]") {
 
 TEST_CASE("ebpps sample: pre-initialized", "[ebpps_sketch]") {
   double theta = 1.0;
-  ebpps_sample<int> sample = ebpps_sample<int>(-1, theta);
+  ebpps_sample<int> sample(1);
+  sample.replace_content(-1, theta);
   REQUIRE(sample.get_c() == theta);
   REQUIRE(sample.get_num_retained_items() == 1);
   REQUIRE(sample.get_sample().size() == 1);
   REQUIRE(sample.has_partial_item() == false);
   
   theta = 1e-300;
-  sample = ebpps_sample<int>(-1, theta);
+  sample.replace_content(-1, theta);
   REQUIRE(sample.get_c() == theta);
   REQUIRE(sample.get_num_retained_items() == 1);
   REQUIRE(sample.get_sample().size() == 0); // assuming the random number is > 1e-300
@@ -57,7 +58,8 @@ TEST_CASE("ebpps sample: pre-initialized", "[ebpps_sketch]") {
 }
 
 TEST_CASE("ebpps sample: downsampling", "[ebpps_sketch]") {
-  ebpps_sample<char> sample = ebpps_sample<char>('a', 1.0);
+  ebpps_sample<char> sample(1);
+  sample.replace_content('a', 1.0);
 
   sample.downsample(2.0); // no-op
   REQUIRE(sample.get_c() == 1.0);
@@ -121,8 +123,9 @@ TEST_CASE("ebpps sample: merge unit samples", "[ebpps_sketch]") {
   uint32_t k = 8;
   ebpps_sample<int> sample = ebpps_sample<int>(k);
   
+  ebpps_sample<int> s(1);
   for (uint32_t i = 1; i <= k; ++i) {
-    ebpps_sample<int> s = ebpps_sample<int>(i, 1.0);
+    s.replace_content(i, 1.0);
     sample.merge(s);
     REQUIRE(sample.get_c() == static_cast<double>(i));
     REQUIRE(sample.get_num_retained_items() == i);

data/vendor/datasketches-cpp/tdigest/CMakeLists.txt

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+add_library(tdigest INTERFACE)
+
+add_library(${PROJECT_NAME}::TDIGEST ALIAS tdigest)
+
+if (BUILD_TESTS)
+  add_subdirectory(test)
+endif()
+
+target_include_directories(tdigest
+  INTERFACE
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+    $<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>
+)
+
+target_link_libraries(tdigest INTERFACE common)
+
+install(TARGETS tdigest
+  EXPORT ${PROJECT_NAME}
+)
+
+install(FILES
+    include/tdigest.hpp
+    include/tdigest_impl.hpp
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")