datasketches 0.2.4 → 0.2.5

data/vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp

@@ -23,6 +23,7 @@
 #include <cstring>
 #include <limits>
 #include <sstream>
+#include <stdexcept>
 
 #include "memory_operations.hpp"
 
@@ -160,7 +161,8 @@ frequent_items_sketch<T, W, H, E, S, A>::get_frequent_items(frequent_items_error
 }
 
 template<typename T, typename W, typename H, typename E, typename S, typename A>
-void frequent_items_sketch<T, W, H, E, S, A>::serialize(std::ostream& os) const {
+template<typename SerDe>
+void frequent_items_sketch<T, W, H, E, S, A>::serialize(std::ostream& os, const SerDe& sd) const {
   const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_EMPTY : PREAMBLE_LONGS_NONEMPTY;
   write(os, preamble_longs);
   const uint8_t serial_version = SERIAL_VERSION;
@@ -198,23 +200,25 @@ void frequent_items_sketch<T, W, H, E, S, A>::serialize(std::ostream& os) const
     }
     write(os, weights, sizeof(W) * num_items);
     aw.deallocate(weights, num_items);
-    S().serialize(os, items, num_items);
+    sd.serialize(os, items, num_items);
     for (i = 0; i < num_items; i++) items[i].~T();
     alloc.deallocate(items, num_items);
   }
 }
 
 template<typename T, typename W, typename H, typename E, typename S, typename A>
-size_t frequent_items_sketch<T, W, H, E, S, A>::get_serialized_size_bytes() const {
+template<typename SerDe>
+size_t frequent_items_sketch<T, W, H, E, S, A>::get_serialized_size_bytes(const SerDe& sd) const {
   if (is_empty()) return PREAMBLE_LONGS_EMPTY * sizeof(uint64_t);
   size_t size = PREAMBLE_LONGS_NONEMPTY * sizeof(uint64_t) + map.get_num_active() * sizeof(W);
-  for (auto it: map) size += S().size_of_item(it.first);
+  for (auto it: map) size += sd.size_of_item(it.first);
   return size;
 }
 
 template<typename T, typename W, typename H, typename E, typename S, typename A>
-auto frequent_items_sketch<T, W, H, E, S, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
-  const size_t size = header_size_bytes + get_serialized_size_bytes();
+template<typename SerDe>
+auto frequent_items_sketch<T, W, H, E, S, A>::serialize(unsigned header_size_bytes, const SerDe& sd) const -> vector_bytes {
+  const size_t size = header_size_bytes + get_serialized_size_bytes(sd);
   vector_bytes bytes(size, 0, map.get_allocator());
   uint8_t* ptr = bytes.data() + header_size_bytes;
   uint8_t* end_ptr = ptr + size;
@@ -255,7 +259,7 @@ auto frequent_items_sketch<T, W, H, E, S, A>::serialize(unsigned header_size_byt
     ptr += copy_to_mem(weights, ptr, sizeof(W) * num_items);
     aw.deallocate(weights, num_items);
     const size_t bytes_remaining = end_ptr - ptr;
-    ptr += S().serialize(ptr, bytes_remaining, items, num_items);
+    ptr += sd.serialize(ptr, bytes_remaining, items, num_items);
     for (i = 0; i < num_items; i++) items[i].~T();
     alloc.deallocate(items, num_items);
   }
@@ -284,6 +288,12 @@ private:
 
 template<typename T, typename W, typename H, typename E, typename S, typename A>
 frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>::deserialize(std::istream& is, const A& allocator) {
+  return deserialize(is, S(), allocator);
+}
+
+template<typename T, typename W, typename H, typename E, typename S, typename A>
+template<typename SerDe>
+frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>::deserialize(std::istream& is, const SerDe& sd, const A& allocator) {
   const auto preamble_longs = read<uint8_t>(is);
   const auto serial_version = read<uint8_t>(is);
   const auto family_id = read<uint8_t>(is);
@@ -312,7 +322,7 @@ frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>:
     read(is, weights.data(), sizeof(W) * num_items);
     A alloc(allocator);
     std::unique_ptr<T, items_deleter> items(alloc.allocate(num_items), items_deleter(num_items, false, alloc));
-    S().deserialize(is, items.get(), num_items);
+    sd.deserialize(is, items.get(), num_items);
     items.get_deleter().set_destroy(true); // serde did not throw, so the items must be constructed
     for (uint32_t i = 0; i < num_items; i++) {
       sketch.update(std::move(items.get()[i]), weights[i]);
@@ -327,6 +337,12 @@ frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>:
 
 template<typename T, typename W, typename H, typename E, typename S, typename A>
 frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>::deserialize(const void* bytes, size_t size, const A& allocator) {
+  return deserialize(bytes, size, S(), allocator);
+}
+
+template<typename T, typename W, typename H, typename E, typename S, typename A>
+template<typename SerDe>
+frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>::deserialize(const void* bytes, size_t size, const SerDe& sd, const A& allocator) {
   ensure_minimum_memory(size, 8);
   const char* ptr = static_cast<const char*>(bytes);
   const char* base = static_cast<const char*>(bytes);
@@ -370,7 +386,7 @@ frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>:
     A alloc(allocator);
     std::unique_ptr<T, items_deleter> items(alloc.allocate(num_items), items_deleter(num_items, false, alloc));
     const size_t bytes_remaining = size - (ptr - base);
-    ptr += S().deserialize(ptr, bytes_remaining, items.get(), num_items);
+    ptr += sd.deserialize(ptr, bytes_remaining, items.get(), num_items);
     items.get_deleter().set_destroy(true); // serde did not throw, so the items must be constructed
     for (uint32_t i = 0; i < num_items; i++) {
       sketch.update(std::move(items.get()[i]), weights[i]);

data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_custom_type_test.cpp

@@ -19,6 +19,7 @@
 
 #include <catch.hpp>
 #include <sstream>
+#include <stdexcept>
 
 #include "frequent_items_sketch.hpp"
 #include "test_type.hpp"
@@ -59,7 +60,7 @@ TEST_CASE("frequent items: custom type", "[frequent_items_sketch]") {
   REQUIRE(sketch.get_maximum_error() == sketch2.get_maximum_error());
 
   auto bytes = sketch.serialize();
-  auto sketch3 = frequent_test_type_sketch::deserialize(bytes.data(), bytes.size(), 0);
+  auto sketch3 = frequent_test_type_sketch::deserialize(bytes.data(), bytes.size(), alloc(0));
   REQUIRE_FALSE(sketch3.is_empty());
   REQUIRE(sketch3.get_total_weight() == 17);
   REQUIRE(sketch3.get_estimate(1) == 10);

data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_test.cpp

@@ -20,6 +20,7 @@
 #include <catch.hpp>
 #include <sstream>
 #include <fstream>
+#include <stdexcept>
 
 #include "frequent_items_sketch.hpp"
 

data/vendor/datasketches-cpp/hll/include/AuxHashMap-internal.hpp

@@ -20,6 +20,8 @@
 #ifndef _AUXHASHMAP_INTERNAL_HPP_
 #define _AUXHASHMAP_INTERNAL_HPP_
 
+#include <stdexcept>
+
 #include "HllUtil.hpp"
 #include "AuxHashMap.hpp"
 

data/vendor/datasketches-cpp/hll/include/CompositeInterpolationXTable-internal.hpp

@@ -24,6 +24,7 @@
 #include "CompositeInterpolationXTable.hpp"
 
 #include <exception>
+#include <stdexcept>
 
 namespace datasketches {
 

data/vendor/datasketches-cpp/hll/include/CouponHashSet-internal.hpp

@@ -24,6 +24,7 @@
 
 #include <cstring>
 #include <exception>
+#include <stdexcept>
 
 namespace datasketches {
 
@@ -113,10 +114,9 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(const void* bytes, size_t len, const
   } else {
     sketch->coupons_.resize(1ULL << lgArrInts);
     sketch->couponCount_ = couponCount;
-    // only need to read valid coupons, unlike in stream case
     std::memcpy(sketch->coupons_.data(),
                 data + hll_constants::HASH_SET_INT_ARR_START,
-                couponCount * sizeof(uint32_t));
+                couponsInArray * sizeof(uint32_t));
   }
 
   return sketch;

data/vendor/datasketches-cpp/hll/include/CouponList-internal.hpp

@@ -27,6 +27,7 @@
 
 #include <algorithm>
 #include <cmath>
+#include <stdexcept>
 
 namespace datasketches {
 

data/vendor/datasketches-cpp/hll/include/HllSketchImpl-internal.hpp

@@ -23,6 +23,8 @@
 #include "HllSketchImpl.hpp"
 #include "HllSketchImplFactory.hpp"
 
+#include <stdexcept>
+
 namespace datasketches {
 
 template<typename A>

data/vendor/datasketches-cpp/hll/include/HllSketchImplFactory.hpp

@@ -20,6 +20,8 @@
 #ifndef _HLLSKETCHIMPLFACTORY_HPP_
 #define _HLLSKETCHIMPLFACTORY_HPP_
 
+#include <stdexcept>
+
 #include "HllUtil.hpp"
 #include "HllSketchImpl.hpp"
 #include "CouponList.hpp"

data/vendor/datasketches-cpp/hll/test/AuxHashMapTest.cpp

@@ -19,6 +19,7 @@
 
 #include <catch.hpp>
 #include <memory>
+#include <stdexcept>
 
 #include "AuxHashMap.hpp"
 

data/vendor/datasketches-cpp/hll/test/CouponHashSetTest.cpp

@@ -26,6 +26,7 @@
 #include <cmath>
 #include <string>
 #include <exception>
+#include <stdexcept>
 
 namespace datasketches {
 

data/vendor/datasketches-cpp/hll/test/CouponListTest.cpp

@@ -23,6 +23,7 @@
 #include <cmath>
 #include <string>
 #include <exception>
+#include <stdexcept>
 
 #include "hll.hpp"
 #include "CouponList.hpp"
@@ -31,6 +32,7 @@
 namespace datasketches {
 
 void println_string(std::string str) {
+  unused(str);
   //std::cout << str << std::endl;
 }
 

data/vendor/datasketches-cpp/hll/test/HllArrayTest.cpp

@@ -21,6 +21,7 @@
 
 #include <exception>
 #include <sstream>
+#include <stdexcept>
 #include <catch.hpp>
 
 namespace datasketches {

data/vendor/datasketches-cpp/hll/test/HllSketchTest.cpp

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+#include <stdexcept>
+
 #include "hll.hpp"
 
 #include <catch.hpp>
@@ -382,4 +384,61 @@ TEST_CASE("hll sketch: deserialize HLL mode buffer overrun", "[hll_sketch]") {
   REQUIRE(test_allocator_total_bytes == 0);
 }
 
+TEST_CASE("hll sketch: bytes serialize-deserialize-serialize list mode") {
+  test_allocator_total_bytes = 0;
+  {
+    hll_sketch_test_alloc s1(10, target_hll_type::HLL_4, false, 0);
+    s1.update(1);
+    s1.update(2);
+    s1.update(3);
+    std::cout << s1.to_string();
+    auto bytes1 = s1.serialize_compact();
+    auto s2 = hll_sketch_test_alloc::deserialize(bytes1.data(), bytes1.size(), 0);
+    auto bytes2 = s2.serialize_compact();
+    REQUIRE(bytes1 == bytes2);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+}
+
+TEST_CASE("hll sketch: updatable bytes serialize-deserialize-serialize set mode") {
+  test_allocator_total_bytes = 0;
+  {
+    hll_sketch_test_alloc s1(10, target_hll_type::HLL_4, false, 0);
+    for (int i = 0; i < 10; ++i) s1.update(i);
+    std::cout << s1.to_string();
+    auto bytes1 = s1.serialize_updatable();
+    auto s2 = hll_sketch_test_alloc::deserialize(bytes1.data(), bytes1.size(), 0);
+
+    auto bytes2 = s2.serialize_updatable();
+    REQUIRE(bytes1 == bytes2);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+}
+
+TEST_CASE("hll sketch: compact bytes serialize-deserialize-serialize set mode") {
+  test_allocator_total_bytes = 0;
+  {
+    hll_sketch_test_alloc s1(10, target_hll_type::HLL_4, false, 0);
+    for (int i = 0; i < 10; ++i) s1.update(i);
+    std::cout << s1.to_string();
+    auto bytes1 = s1.serialize_compact();
+    auto s2 = hll_sketch_test_alloc::deserialize(bytes1.data(), bytes1.size(), 0);
+
+    // cannot just compare bytes here
+    // hash set does not preserve the order after reconstruction in compact mode
+    // add more to push them to HLL mode
+    for (int i = 10; i < 100; ++i) {
+      s1.update(i);
+      s2.update(i);
+    }
+    std::cout << s1.to_string();
+    std::cout << s2.to_string();
+
+    auto bytes2 = s1.serialize_compact();
+    auto bytes3 = s2.serialize_compact();
+    REQUIRE(bytes2 == bytes3);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp

@@ -19,12 +19,14 @@
 
 #include <catch.hpp>
 #include <sstream>
+#include <stdexcept>
 
 #include "hll.hpp"
 
 namespace datasketches {
 
 static void println(std::string& str) {
+  unused(str);
   //std::cout << str << "\n";
 }
 

data/vendor/datasketches-cpp/hll/test/TablesTest.cpp

@@ -17,6 +17,7 @@
  * under the License.
  */
 
+#include <stdexcept>
 
 #include <catch.hpp>
 

data/vendor/datasketches-cpp/kll/CMakeLists.txt

@@ -41,8 +41,4 @@ install(FILES
 		include/kll_sketch_impl.hpp	
 		include/kll_helper.hpp
 		include/kll_helper_impl.hpp
-		include/kll_quantile_calculator.hpp
-		include/kll_quantile_calculator_impl.hpp
-		include/kolmogorov_smirnov.hpp
-		include/kolmogorov_smirnov_impl.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")

data/vendor/datasketches-cpp/kll/include/kll_helper.hpp

@@ -22,13 +22,9 @@
 
 #include <random>
 #include <stdexcept>
-#include <chrono>
 
 namespace datasketches {
 
-static std::independent_bits_engine<std::mt19937, 1, uint32_t>
-  random_bit(static_cast<uint32_t>(std::chrono::system_clock::now().time_since_epoch().count()));
-
 #ifdef KLL_VALIDATION
 extern uint32_t kll_next_offset;
 #endif

data/vendor/datasketches-cpp/kll/include/kll_helper_impl.hpp

@@ -21,6 +21,9 @@
 #define KLL_HELPER_IMPL_HPP_
 
 #include <algorithm>
+#include <stdexcept>
+
+#include "common_defs.hpp"
 
 namespace datasketches {