datasketches 0.2.3 → 0.2.6

data/vendor/datasketches-cpp/hll/CMakeLists.txt

@@ -32,64 +32,41 @@ target_include_directories(hll
 target_link_libraries(hll INTERFACE common)
 target_compile_features(hll INTERFACE cxx_std_11)
 
-# TODO: would be useful if this didn't need to be reproduced in target_sources(), too
-set(hll_HEADERS "")
-list(APPEND hll_HEADERS "include/hll.hpp;include/AuxHashMap.hpp;include/CompositeInterpolationXTable.hpp")
-list(APPEND hll_HEADERS "include/hll.private.hpp;include/HllSketchImplFactory.hpp")
-list(APPEND hll_HEADERS "include/CouponHashSet.hpp;include/CouponList.hpp")
-list(APPEND hll_HEADERS "include/CubicInterpolation.hpp;include/HarmonicNumbers.hpp;include/Hll4Array.hpp")
-list(APPEND hll_HEADERS "include/Hll6Array.hpp;include/Hll8Array.hpp;include/HllArray.hpp")
-list(APPEND hll_HEADERS "include/HllSketchImpl.hpp")
-list(APPEND hll_HEADERS "include/HllUtil.hpp;include/coupon_iterator.hpp")
-list(APPEND hll_HEADERS "include/RelativeErrorTables.hpp;include/AuxHashMap-internal.hpp")
-list(APPEND hll_HEADERS "include/CompositeInterpolationXTable-internal.hpp")
-list(APPEND hll_HEADERS "include/CouponHashSet-internal.hpp;include/CouponList-internal.hpp")
-list(APPEND hll_HEADERS "include/CubicInterpolation-internal.hpp;include/HarmonicNumbers-internal.hpp")
-list(APPEND hll_HEADERS "include/Hll4Array-internal.hpp;include/Hll6Array-internal.hpp")
-list(APPEND hll_HEADERS "include/Hll8Array-internal.hpp;include/HllArray-internal.hpp")
-list(APPEND hll_HEADERS "include/HllSketch-internal.hpp")
-list(APPEND hll_HEADERS "include/HllSketchImpl-internal.hpp;include/HllUnion-internal.hpp")
-list(APPEND hll_HEADERS "include/coupon_iterator-internal.hpp;include/RelativeErrorTables-internal.hpp")
-
 install(TARGETS hll
   EXPORT ${PROJECT_NAME}
 )
 
-install(FILES ${hll_HEADERS}
+install(FILES 
+			include/hll.hpp
+			include/AuxHashMap.hpp
+			include/CompositeInterpolationXTable.hpp
+			include/hll.private.hpp
+			include/HllSketchImplFactory.hpp
+			include/CouponHashSet.hpp
+			include/CouponList.hpp
+			include/CubicInterpolation.hpp
+			include/HarmonicNumbers.hpp
+			include/Hll4Array.hpp
+			include/Hll6Array.hpp
+			include/Hll8Array.hpp
+			include/HllArray.hpp
+			include/HllSketchImpl.hpp
+			include/HllUtil.hpp
+			include/coupon_iterator.hpp
+			include/RelativeErrorTables.hpp
+			include/AuxHashMap-internal.hpp
+			include/CompositeInterpolationXTable-internal.hpp
+			include/CouponHashSet-internal.hpp
+			include/CouponList-internal.hpp
+			include/CubicInterpolation-internal.hpp
+			include/HarmonicNumbers-internal.hpp
+			include/Hll4Array-internal.hpp
+			include/Hll6Array-internal.hpp
+			include/Hll8Array-internal.hpp
+			include/HllArray-internal.hpp
+			include/HllSketch-internal.hpp
+			include/HllSketchImpl-internal.hpp
+			include/HllUnion-internal.hpp
+			include/coupon_iterator-internal.hpp
+			include/RelativeErrorTables-internal.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
-
-target_sources(hll
-  INTERFACE
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/hll.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/hll.private.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/AuxHashMap.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/CompositeInterpolationXTable.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/CouponHashSet.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/CouponList.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/CubicInterpolation.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/HarmonicNumbers.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/Hll4Array.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/Hll6Array.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/Hll8Array.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/HllArray.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/HllSketchImpl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/HllSketchImplFactory.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/HllUtil.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/RelativeErrorTables.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/coupon_iterator.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/AuxHashMap-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/CompositeInterpolationXTable-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/CouponHashSet-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/CouponList-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/CubicInterpolation-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/HarmonicNumbers-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/Hll4Array-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/Hll6Array-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/Hll8Array-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/HllArray-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/HllSketch-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/HllSketchImpl-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/HllUnion-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/RelativeErrorTables-internal.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/coupon_iterator-internal.hpp
-)

data/vendor/datasketches-cpp/hll/include/AuxHashMap-internal.hpp

@@ -20,6 +20,8 @@
 #ifndef _AUXHASHMAP_INTERNAL_HPP_
 #define _AUXHASHMAP_INTERNAL_HPP_
 
+#include <stdexcept>
+
 #include "HllUtil.hpp"
 #include "AuxHashMap.hpp"
 

data/vendor/datasketches-cpp/hll/include/CompositeInterpolationXTable-internal.hpp

@@ -24,6 +24,7 @@
 #include "CompositeInterpolationXTable.hpp"
 
 #include <exception>
+#include <stdexcept>
 
 namespace datasketches {
 

data/vendor/datasketches-cpp/hll/include/CouponHashSet-internal.hpp

@@ -24,6 +24,7 @@
 
 #include <cstring>
 #include <exception>
+#include <stdexcept>
 
 namespace datasketches {
 
@@ -113,10 +114,9 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(const void* bytes, size_t len, const
   } else {
     sketch->coupons_.resize(1ULL << lgArrInts);
     sketch->couponCount_ = couponCount;
-    // only need to read valid coupons, unlike in stream case
     std::memcpy(sketch->coupons_.data(),
                 data + hll_constants::HASH_SET_INT_ARR_START,
-                couponCount * sizeof(uint32_t));
+                couponsInArray * sizeof(uint32_t));
   }
 
   return sketch;

data/vendor/datasketches-cpp/hll/include/CouponList-internal.hpp

@@ -27,6 +27,7 @@
 
 #include <algorithm>
 #include <cmath>
+#include <stdexcept>
 
 namespace datasketches {
 

data/vendor/datasketches-cpp/hll/include/HllSketch-internal.hpp

@@ -246,10 +246,12 @@ string<A> hll_sketch_alloc<A>::to_string(const bool summary,
                                          const bool detail,
                                          const bool aux_detail,
                                          const bool all) const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::stringstream os;
   if (summary) {
     os << "### HLL sketch summary:" << std::endl
-       << "  Log Config K   : " << get_lg_config_k() << std::endl
+       << "  Log Config K   : " << std::to_string(get_lg_config_k()) << std::endl
        << "  Hll Target     : " << type_as_string() << std::endl
        << "  Current Mode   : " << mode_as_string() << std::endl
        << "  LB             : " << get_lower_bound(1) << std::endl
@@ -258,7 +260,7 @@ string<A> hll_sketch_alloc<A>::to_string(const bool summary,
        << "  OutOfOrder flag: " << (is_out_of_order_flag() ? "true" : "false") << std::endl;
     if (get_current_mode() == HLL) {
       HllArray<A>* hllArray = (HllArray<A>*) sketch_impl;
-      os << "  CurMin         : " << hllArray->getCurMin() << std::endl
+      os << "  CurMin         : " << std::to_string(hllArray->getCurMin()) << std::endl
          << "  NumAtCurMin    : " << hllArray->getNumAtCurMin() << std::endl
          << "  HipAccum       : " << hllArray->getHipAccum() << std::endl
          << "  KxQ0           : " << hllArray->getKxQ0() << std::endl
@@ -338,7 +340,7 @@ string<A> hll_sketch_alloc<A>::to_string(const bool summary,
     }
   }
 
-  return os.str();
+  return string<A>(os.str().c_str(), sketch_impl->getAllocator());
 }
 
 template<typename A>

data/vendor/datasketches-cpp/hll/include/HllSketchImpl-internal.hpp

@@ -23,6 +23,8 @@
 #include "HllSketchImpl.hpp"
 #include "HllSketchImplFactory.hpp"
 
+#include <stdexcept>
+
 namespace datasketches {
 
 template<typename A>

data/vendor/datasketches-cpp/hll/include/HllSketchImplFactory.hpp

@@ -20,6 +20,8 @@
 #ifndef _HLLSKETCHIMPLFACTORY_HPP_
 #define _HLLSKETCHIMPLFACTORY_HPP_
 
+#include <stdexcept>
+
 #include "HllUtil.hpp"
 #include "HllSketchImpl.hpp"
 #include "CouponList.hpp"

data/vendor/datasketches-cpp/hll/test/AuxHashMapTest.cpp

@@ -19,6 +19,7 @@
 
 #include <catch.hpp>
 #include <memory>
+#include <stdexcept>
 
 #include "AuxHashMap.hpp"
 

data/vendor/datasketches-cpp/hll/test/CouponHashSetTest.cpp

@@ -26,6 +26,7 @@
 #include <cmath>
 #include <string>
 #include <exception>
+#include <stdexcept>
 
 namespace datasketches {
 

data/vendor/datasketches-cpp/hll/test/CouponListTest.cpp

@@ -23,6 +23,7 @@
 #include <cmath>
 #include <string>
 #include <exception>
+#include <stdexcept>
 
 #include "hll.hpp"
 #include "CouponList.hpp"
@@ -31,6 +32,7 @@
 namespace datasketches {
 
 void println_string(std::string str) {
+  unused(str);
   //std::cout << str << std::endl;
 }
 

data/vendor/datasketches-cpp/hll/test/HllArrayTest.cpp

@@ -21,6 +21,7 @@
 
 #include <exception>
 #include <sstream>
+#include <stdexcept>
 #include <catch.hpp>
 
 namespace datasketches {

data/vendor/datasketches-cpp/hll/test/HllSketchTest.cpp

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+#include <stdexcept>
+
 #include "hll.hpp"
 
 #include <catch.hpp>
@@ -382,4 +384,61 @@ TEST_CASE("hll sketch: deserialize HLL mode buffer overrun", "[hll_sketch]") {
   REQUIRE(test_allocator_total_bytes == 0);
 }
 
+TEST_CASE("hll sketch: bytes serialize-deserialize-serialize list mode") {
+  test_allocator_total_bytes = 0;
+  {
+    hll_sketch_test_alloc s1(10, target_hll_type::HLL_4, false, 0);
+    s1.update(1);
+    s1.update(2);
+    s1.update(3);
+    std::cout << s1.to_string();
+    auto bytes1 = s1.serialize_compact();
+    auto s2 = hll_sketch_test_alloc::deserialize(bytes1.data(), bytes1.size(), 0);
+    auto bytes2 = s2.serialize_compact();
+    REQUIRE(bytes1 == bytes2);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+}
+
+TEST_CASE("hll sketch: updatable bytes serialize-deserialize-serialize set mode") {
+  test_allocator_total_bytes = 0;
+  {
+    hll_sketch_test_alloc s1(10, target_hll_type::HLL_4, false, 0);
+    for (int i = 0; i < 10; ++i) s1.update(i);
+    std::cout << s1.to_string();
+    auto bytes1 = s1.serialize_updatable();
+    auto s2 = hll_sketch_test_alloc::deserialize(bytes1.data(), bytes1.size(), 0);
+
+    auto bytes2 = s2.serialize_updatable();
+    REQUIRE(bytes1 == bytes2);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+}
+
+TEST_CASE("hll sketch: compact bytes serialize-deserialize-serialize set mode") {
+  test_allocator_total_bytes = 0;
+  {
+    hll_sketch_test_alloc s1(10, target_hll_type::HLL_4, false, 0);
+    for (int i = 0; i < 10; ++i) s1.update(i);
+    std::cout << s1.to_string();
+    auto bytes1 = s1.serialize_compact();
+    auto s2 = hll_sketch_test_alloc::deserialize(bytes1.data(), bytes1.size(), 0);
+
+    // cannot just compare bytes here
+    // hash set does not preserve the order after reconstruction in compact mode
+    // add more to push them to HLL mode
+    for (int i = 10; i < 100; ++i) {
+      s1.update(i);
+      s2.update(i);
+    }
+    std::cout << s1.to_string();
+    std::cout << s2.to_string();
+
+    auto bytes2 = s1.serialize_compact();
+    auto bytes3 = s2.serialize_compact();
+    REQUIRE(bytes2 == bytes3);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp

@@ -19,12 +19,14 @@
 
 #include <catch.hpp>
 #include <sstream>
+#include <stdexcept>
 
 #include "hll.hpp"
 
 namespace datasketches {
 
 static void println(std::string& str) {
+  unused(str);
   //std::cout << str << "\n";
 }
 

data/vendor/datasketches-cpp/hll/test/TablesTest.cpp

@@ -17,6 +17,7 @@
  * under the License.
  */
 
+#include <stdexcept>
 
 #include <catch.hpp>
 

data/vendor/datasketches-cpp/kll/CMakeLists.txt

@@ -32,27 +32,13 @@ target_include_directories(kll
 target_link_libraries(kll INTERFACE common)
 target_compile_features(kll INTERFACE cxx_std_11)
 
-set(kll_HEADERS "")
-list(APPEND kll_HEADERS "include/kll_sketch.hpp")
-list(APPEND kll_HEADERS "include/kll_sketch_impl.hpp")
-list(APPEND kll_HEADERS "include/kll_helper.hpp")
-list(APPEND kll_HEADERS "include/kll_helper_impl.hpp")
-list(APPEND kll_HEADERS "include/kll_quantile_calculator.hpp")
-list(APPEND kll_HEADERS "include/kll_quantile_calculator_impl.hpp")
-
 install(TARGETS kll
   EXPORT ${PROJECT_NAME}
 )
 
-install(FILES ${kll_HEADERS}
+install(FILES 
+		include/kll_sketch.hpp
+		include/kll_sketch_impl.hpp	
+		include/kll_helper.hpp
+		include/kll_helper_impl.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
-
-target_sources(kll
-  INTERFACE
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_helper.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_helper_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_sketch.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_sketch_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_quantile_calculator.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_quantile_calculator_impl.hpp
-)

data/vendor/datasketches-cpp/kll/include/kll_helper.hpp

@@ -22,13 +22,9 @@
 
 #include <random>
 #include <stdexcept>
-#include <chrono>
 
 namespace datasketches {
 
-static std::independent_bits_engine<std::mt19937, 1, uint32_t>
-  random_bit(static_cast<uint32_t>(std::chrono::system_clock::now().time_since_epoch().count()));
-
 #ifdef KLL_VALIDATION
 extern uint32_t kll_next_offset;
 #endif

data/vendor/datasketches-cpp/kll/include/kll_helper_impl.hpp

@@ -21,6 +21,9 @@
 #define KLL_HELPER_IMPL_HPP_
 
 #include <algorithm>
+#include <stdexcept>
+
+#include "common_defs.hpp"
 
 namespace datasketches {
 
@@ -227,7 +230,7 @@ kll_helper::compress_result kll_helper::general_compress(uint16_t k, uint8_t m,
       // move level over as is
       // make sure we are not moving data upwards
       if (raw_beg < out_levels[current_level]) throw std::logic_error("wrong move");
-      std::move(&items[raw_beg], &items[raw_lim], &items[out_levels[current_level]]);
+      std::move(items + raw_beg, items + raw_lim, items + out_levels[current_level]);
       out_levels[current_level + 1] = out_levels[current_level] + raw_pop;
     } else {
       // The sketch is too full AND this level is too full, so we compact it
@@ -248,7 +251,7 @@ kll_helper::compress_result kll_helper::general_compress(uint16_t k, uint8_t m,
 
       // level zero might not be sorted, so we must sort it if we wish to compact it
       if ((current_level == 0) && !is_level_zero_sorted) {
-        std::sort(&items[adj_beg], &items[adj_beg + adj_pop], C());
+        std::sort(items + adj_beg, items + adj_beg + adj_pop, C());
       }
 
       if (pop_above == 0) { // Level above is empty, so halve up