datasketches 0.2.3 → 0.2.4

data/vendor/datasketches-cpp/kll/CMakeLists.txt

@@ -32,27 +32,17 @@ target_include_directories(kll
 target_link_libraries(kll INTERFACE common)
 target_compile_features(kll INTERFACE cxx_std_11)
 
-set(kll_HEADERS "")
-list(APPEND kll_HEADERS "include/kll_sketch.hpp")
-list(APPEND kll_HEADERS "include/kll_sketch_impl.hpp")
-list(APPEND kll_HEADERS "include/kll_helper.hpp")
-list(APPEND kll_HEADERS "include/kll_helper_impl.hpp")
-list(APPEND kll_HEADERS "include/kll_quantile_calculator.hpp")
-list(APPEND kll_HEADERS "include/kll_quantile_calculator_impl.hpp")
-
 install(TARGETS kll
   EXPORT ${PROJECT_NAME}
 )
 
-install(FILES ${kll_HEADERS}
+install(FILES 
+		include/kll_sketch.hpp
+		include/kll_sketch_impl.hpp	
+		include/kll_helper.hpp
+		include/kll_helper_impl.hpp
+		include/kll_quantile_calculator.hpp
+		include/kll_quantile_calculator_impl.hpp
+		include/kolmogorov_smirnov.hpp
+		include/kolmogorov_smirnov_impl.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
-
-target_sources(kll
-  INTERFACE
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_helper.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_helper_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_sketch.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_sketch_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_quantile_calculator.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/kll_quantile_calculator_impl.hpp
-)

data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp

@@ -1023,7 +1023,9 @@ void kll_sketch<T, C, S, A>::check_family_id(uint8_t family_id) {
 
 template <typename T, typename C, typename S, typename A>
 string<A> kll_sketch<T, C, S, A>::to_string(bool print_levels, bool print_items) const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
   os << "### KLL sketch summary:" << std::endl;
   os << "   K              : " << k_ << std::endl;
   os << "   min K          : " << min_k_ << std::endl;
@@ -1069,7 +1071,7 @@ string<A> kll_sketch<T, C, S, A>::to_string(bool print_levels, bool print_items)
     }
     os << "### End sketch data" << std::endl;
   }
-  return os.str();
+  return string<A>(os.str().c_str(), allocator_);
 }
 
 template <typename T, typename C, typename S, typename A>

data/vendor/datasketches-cpp/req/CMakeLists.txt

@@ -32,29 +32,16 @@ target_include_directories(req
 target_link_libraries(req INTERFACE common)
 target_compile_features(req INTERFACE cxx_std_11)
 
-set(req_HEADERS "")
-list(APPEND req_HEADERS "include/req_common.hpp")
-list(APPEND req_HEADERS "include/req_sketch.hpp")
-list(APPEND req_HEADERS "include/req_sketch_impl.hpp")
-list(APPEND req_HEADERS "include/req_compactor.hpp")
-list(APPEND req_HEADERS "include/req_compactor_impl.hpp")
-list(APPEND req_HEADERS "include/req_quantile_calculator.hpp")
-list(APPEND req_HEADERS "include/req_quantile_calculator_impl.hpp")
-
 install(TARGETS req
   EXPORT ${PROJECT_NAME}
 )
 
-install(FILES ${req_HEADERS}
+install(FILES 
+		include/req_common.hpp
+		include/req_sketch.hpp
+		include/req_sketch_impl.hpp
+		include/req_compactor.hpp
+		include/req_compactor_impl.hpp
+		include/req_quantile_calculator.hpp
+		include/req_quantile_calculator_impl.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
-
-target_sources(req
-  INTERFACE
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/req_common.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/req_sketch.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/req_sketch_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/req_compactor.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/req_compactor_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/req_quantile_calculator.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/req_quantile_calculator_impl.hpp
-)

data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp

@@ -653,7 +653,9 @@ void req_sketch<T, C, S, A>::compress() {
 
 template<typename T, typename C, typename S, typename A>
 string<A> req_sketch<T, C, S, A>::to_string(bool print_levels, bool print_items) const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
   os << "### REQ sketch summary:" << std::endl;
   os << "   K              : " << k_ << std::endl;
   os << "   High Rank Acc  : " << (hra_ ? "true" : "false") << std::endl;
@@ -693,7 +695,7 @@ string<A> req_sketch<T, C, S, A>::to_string(bool print_levels, bool print_items)
     }
     os << "### End sketch data" << std::endl;
   }
-  return os.str();
+  return string<A>(os.str().c_str(), allocator_);
 }
 
 template<typename T, typename C, typename S, typename A>

data/vendor/datasketches-cpp/sampling/CMakeLists.txt

@@ -32,17 +32,13 @@ target_include_directories(sampling
 target_link_libraries(sampling INTERFACE common)
 target_compile_features(sampling INTERFACE cxx_std_11)
 
-set(sampling_HEADERS "include/var_opt_sketch.hpp;include/var_opt_sketch_impl.hpp")
-
 install(TARGETS sampling
   EXPORT ${PROJECT_NAME}
 )
 
-install(FILES ${sampling_HEADERS}
+install(FILES 
+		include/var_opt_sketch.hpp
+		include/var_opt_sketch_impl.hpp
+		include/var_opt_union.hpp
+		include/var_opt_union_impl.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
-
-target_sources(sampling
-  INTERFACE
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/var_opt_sketch.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/var_opt_sketch_impl.hpp
-)

data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp

@@ -731,8 +731,10 @@ void var_opt_sketch<T,S,A>::update(T&& item, double weight) {
 
 template<typename T, typename S, typename A>
 string<A> var_opt_sketch<T,S,A>::to_string() const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
-  os << "### VarOpt SUMMARY: " << std::endl;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
+  os << "### VarOpt SUMMARY:" << std::endl;
   os << "   k            : " << k_ << std::endl;
   os << "   h            : " << h_ << std::endl;
   os << "   r            : " << r_ << std::endl;
@@ -740,24 +742,28 @@ string<A> var_opt_sketch<T,S,A>::to_string() const {
   os << "   Current size : " << curr_items_alloc_ << std::endl;
   os << "   Resize factor: " << (1 << rf_) << std::endl;
   os << "### END SKETCH SUMMARY" << std::endl;
-  return os.str();
+  return string<A>(os.str().c_str(), allocator_);
 }
 
 template<typename T, typename S, typename A>
 string<A> var_opt_sketch<T,S,A>::items_to_string() const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
   os << "### Sketch Items" << std::endl;
   int idx = 0;
   for (auto record : *this) {
     os << idx << ": " << record.first << "\twt = " << record.second << std::endl;
     ++idx;
   }
-  return os.str();
+  return string<A>(os.str().c_str(), allocator_);
 }
 
 template<typename T, typename S, typename A>
 string<A> var_opt_sketch<T,S,A>::items_to_string(bool print_gap) const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
   os << "### Sketch Items" << std::endl;
   const uint32_t array_length = (n_ < k_ ? n_ : k_ + 1);
   for (uint32_t i = 0, display_idx = 0; i < array_length; ++i) {
@@ -774,7 +780,7 @@ string<A> var_opt_sketch<T,S,A>::items_to_string(bool print_gap) const {
       ++display_idx;
     }
   }
-  return os.str();
+  return string<A>(os.str().c_str(), allocator_);
 }
 
 template<typename T, typename S, typename A>

data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp

@@ -295,14 +295,16 @@ void var_opt_union<T,S,A>::reset() {
 
 template<typename T, typename S, typename A>
 string<A> var_opt_union<T,S,A>::to_string() const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
-  os << "### VarOpt Union SUMMARY: " << std::endl;
-  os << " . n             : " << n_ << std::endl;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
+  os << "### VarOpt Union SUMMARY:" << std::endl;
+  os << "   n             : " << n_ << std::endl;
   os << "   Max k         : " << max_k_ << std::endl;
-  os << "   Gadget Summary: " << std::endl;
+  os << "   Gadget Summary:" << std::endl;
   os << gadget_.to_string();
-  os << "### END VarOpt Union SUMMARY: " << std::endl;
-  return os.str();
+  os << "### END VarOpt Union SUMMARY" << std::endl;
+  return string<A>(os.str().c_str(), gadget_.allocator_);
 }
 
 template<typename T, typename S, typename A>

data/vendor/datasketches-cpp/setup.py

@@ -81,7 +81,7 @@ class CMakeBuild(build_ext):
 
 setup(
     name='datasketches',
-    version='3.2.0.1',
+    version='3.3.0',
     author='Apache Software Foundation',
     author_email='dev@datasketches.apache.org',
     description='The Apache DataSketches Library for Python',

data/vendor/datasketches-cpp/theta/CMakeLists.txt

@@ -32,53 +32,34 @@ target_include_directories(theta
 target_link_libraries(theta INTERFACE common)
 target_compile_features(theta INTERFACE cxx_std_11)
 
-set(theta_HEADERS "")
-list(APPEND theta_HEADERS "include/theta_sketch.hpp;include/theta_sketch_impl.hpp")
-list(APPEND theta_HEADERS "include/theta_union.hpp;include/theta_union_impl.hpp")
-list(APPEND theta_HEADERS "include/theta_intersection.hpp;include/theta_intersection_impl.hpp")
-list(APPEND theta_HEADERS "include/theta_a_not_b.hpp;include/theta_a_not_b_impl.hpp")
-list(APPEND theta_HEADERS "include/theta_jaccard_similarity.hpp")
-list(APPEND theta_HEADERS "include/theta_comparators.hpp")
-list(APPEND theta_HEADERS "include/theta_constants.hpp")
-list(APPEND theta_HEADERS "include/theta_helpers.hpp")
-list(APPEND theta_HEADERS "include/theta_update_sketch_base.hpp;include/theta_update_sketch_base_impl.hpp")
-list(APPEND theta_HEADERS "include/theta_union_base.hpp;include/theta_union_base_impl.hpp")
-list(APPEND theta_HEADERS "include/theta_intersection_base.hpp;include/theta_intersection_base_impl.hpp")
-list(APPEND theta_HEADERS "include/theta_set_difference_base.hpp;include/theta_set_difference_base_impl.hpp")
-list(APPEND theta_HEADERS "include/theta_jaccard_similarity_base.hpp")
-list(APPEND theta_HEADERS "include/bounds_on_ratios_in_sampled_sets.hpp")
-list(APPEND theta_HEADERS "include/bounds_on_ratios_in_theta_sketched_sets.hpp")
-
 install(TARGETS theta
   EXPORT ${PROJECT_NAME}
 )
 
-install(FILES ${theta_HEADERS}
+install(FILES 
+			include/theta_sketch.hpp
+			include/theta_sketch_impl.hpp
+			include/theta_union.hpp
+			include/theta_union_impl.hpp
+			include/theta_intersection.hpp
+			include/theta_intersection_impl.hpp
+			include/theta_a_not_b.hpp
+			include/theta_a_not_b_impl.hpp
+			include/theta_jaccard_similarity.hpp
+			include/theta_comparators.hpp
+			include/theta_constants.hpp
+			include/theta_helpers.hpp
+			include/theta_update_sketch_base.hpp
+			include/theta_update_sketch_base_impl.hpp
+			include/theta_union_base.hpp
+			include/theta_union_base_impl.hpp
+			include/theta_intersection_base.hpp
+			include/theta_intersection_base_impl.hpp
+			include/theta_set_difference_base.hpp
+			include/theta_set_difference_base_impl.hpp
+			include/theta_jaccard_similarity_base.hpp
+			include/bounds_on_ratios_in_sampled_sets.hpp
+			include/bounds_on_ratios_in_theta_sketched_sets.hpp
+			include/compact_theta_sketch_parser.hpp
+			include/compact_theta_sketch_parser_impl.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
-
-target_sources(theta
-  INTERFACE
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_sketch.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_a_not_b.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_sketch_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_a_not_b_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_jaccard_similarity.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_comparators.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_constants.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_helpers.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_update_sketch_base.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_update_sketch_base_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_base.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_base_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_base.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_base_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_set_difference_base.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_set_difference_base_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_jaccard_similarity_base.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/bounds_on_ratios_in_sampled_sets.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/bounds_on_ratios_in_theta_sketched_sets.hpp
-)

data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser_impl.hpp

@@ -29,32 +29,99 @@ template<bool dummy>
 auto compact_theta_sketch_parser<dummy>::parse(const void* ptr, size_t size, uint64_t seed, bool dump_on_error) -> compact_theta_sketch_data {
   if (size < 8) throw std::invalid_argument("at least 8 bytes expected, actual " + std::to_string(size)
       + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
-  checker<true>::check_serial_version(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE], COMPACT_SKETCH_SERIAL_VERSION);
-  checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
-  uint64_t theta = theta_constants::MAX_THETA;
-  const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
-  checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
-  if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_EMPTY_FLAG)) {
-    return {true, true, seed_hash, 0, theta, nullptr};
+
+  uint8_t serial_version = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE];
+
+  switch(serial_version) {
+  case COMPACT_SKETCH_SERIAL_VERSION: {
+      checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+      uint64_t theta = theta_constants::MAX_THETA;
+      const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
+      if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_EMPTY_FLAG)) {
+        return {true, true, seed_hash, 0, theta, nullptr};
+      }
+      checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+      const bool has_theta = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 2;
+      if (has_theta) {
+        if (size < 16) throw std::invalid_argument("at least 16 bytes expected, actual " + std::to_string(size));
+        theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+      }
+      if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] == 1) {
+        return {false, true, seed_hash, 1, theta, reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_SINGLE_ENTRY_U64};
+      }
+      const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+      const size_t entries_start_u64 = has_theta ? COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 : COMPACT_SKETCH_ENTRIES_EXACT_U64;
+      const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + entries_start_u64;
+      const size_t expected_size_bytes = (entries_start_u64 + num_entries) * sizeof(uint64_t);
+      if (size < expected_size_bytes) {
+        throw std::invalid_argument(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
+            + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
+      }
+      const bool is_ordered = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_ORDERED_FLAG);
+      return {false, is_ordered, seed_hash, num_entries, theta, entries};
   }
-  const bool has_theta = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 2;
-  if (has_theta) {
-    if (size < 16) throw std::invalid_argument("at least 16 bytes expected, actual " + std::to_string(size));
-    theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+  case 1:  {
+      uint16_t seed_hash = compute_seed_hash(seed);
+      checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+      const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+      uint64_t theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+      bool is_empty = (num_entries == 0) && (theta == theta_constants::MAX_THETA);
+      if (is_empty) {
+          return {true, true, seed_hash, 0, theta, nullptr};
+      }
+      const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_ENTRIES_ESTIMATION_U64;
+      const size_t expected_size_bytes = (COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 + num_entries) * sizeof(uint64_t);
+      if (size < expected_size_bytes) {
+        throw std::invalid_argument(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
+            + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
+      }
+      return {false, true, seed_hash, num_entries, theta, entries};
   }
-  if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] == 1) {
-    return {false, true, seed_hash, 1, theta, reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_SINGLE_ENTRY_U64};
+  case 2:  {
+      uint8_t preamble_size =  reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE];
+      checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+      const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
+      checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+      if (preamble_size == 1) {
+          return {true, true, seed_hash, 0, theta_constants::MAX_THETA, nullptr};
+      } else if (preamble_size == 2) {
+          const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+          if (num_entries == 0) {
+              return {true, true, seed_hash, 0, theta_constants::MAX_THETA, nullptr};
+          } else {
+              const size_t expected_size_bytes = (preamble_size + num_entries) << 3;
+              if (size < expected_size_bytes) {
+                  throw std::invalid_argument(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
+                      + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
+              }
+              const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_ENTRIES_EXACT_U64;
+              return {false, true, seed_hash, num_entries, theta_constants::MAX_THETA, entries};
+          }
+      } else if (preamble_size == 3) {
+          const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+          uint64_t theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+          bool is_empty = (num_entries == 0) && (theta == theta_constants::MAX_THETA);
+          if (is_empty) {
+              return {true, true, seed_hash, 0, theta, nullptr};
+          }
+          const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_ENTRIES_ESTIMATION_U64;
+          const size_t expected_size_bytes = (COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 + num_entries) * sizeof(uint64_t);
+          if (size < expected_size_bytes) {
+            throw std::invalid_argument(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
+                + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
+          }
+          return {false, true, seed_hash, num_entries, theta, entries};
+      } else {
+          throw std::invalid_argument(std::to_string(preamble_size) + " longs of premable, but expected 1, 2, or 3");
+      }
   }
-  const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
-  const size_t entries_start_u64 = has_theta ? COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 : COMPACT_SKETCH_ENTRIES_EXACT_U64;
-  const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + entries_start_u64;
-  const size_t expected_size_bytes = (entries_start_u64 + num_entries) * sizeof(uint64_t);
-  if (size < expected_size_bytes) {
-    throw std::invalid_argument(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
-        + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
+  default:
+      // this should always fail since the valid cases are handled above
+      checker<true>::check_serial_version(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE], COMPACT_SKETCH_SERIAL_VERSION);
+      // this throw is never reached, because check_serial_version will throw an informative exception.
+      // This is only here to avoid a compiler warning about a path without a return value.
+      throw std::invalid_argument("unexpected sketch serialization version");
   }
-  const bool is_ordered = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_ORDERED_FLAG);
-  return {false, is_ordered, seed_hash, num_entries, theta, entries};
 }
 
 template<bool dummy>

data/vendor/datasketches-cpp/theta/include/theta_helpers.hpp

@@ -49,6 +49,21 @@ public:
   }
 };
 
+template<bool dummy>
+class theta_build_helper{
+public:
+  // consistent way of initializing theta from p
+  // avoids multiplication if p == 1 since it might not yield MAX_THETA exactly
+  static uint64_t starting_theta_from_p(float p) {
+    if (p < 1) return static_cast<uint64_t>(theta_constants::MAX_THETA * p);
+    return theta_constants::MAX_THETA;
+  }
+
+  static uint8_t starting_sub_multiple(uint8_t lg_tgt, uint8_t lg_min, uint8_t lg_rf) {
+    return (lg_tgt <= lg_min) ? lg_min : (lg_rf == 0) ? lg_tgt : ((lg_tgt - lg_min) % lg_rf) + lg_min;
+  }
+};
+
 } /* namespace datasketches */
 
 #endif

data/vendor/datasketches-cpp/theta/include/theta_intersection_base_impl.hpp

@@ -29,7 +29,7 @@ template<typename EN, typename EK, typename P, typename S, typename CS, typename
 theta_intersection_base<EN, EK, P, S, CS, A>::theta_intersection_base(uint64_t seed, const P& policy, const A& allocator):
 policy_(policy),
 is_valid_(false),
-table_(0, 0, resize_factor::X1, theta_constants::MAX_THETA, seed, allocator, false)
+table_(0, 0, resize_factor::X1, 1, theta_constants::MAX_THETA, seed, allocator, false)
 {}
 
 template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
@@ -38,17 +38,17 @@ void theta_intersection_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
   if (table_.is_empty_) return;
   if (!sketch.is_empty() && sketch.get_seed_hash() != compute_seed_hash(table_.seed_)) throw std::invalid_argument("seed hash mismatch");
   table_.is_empty_ |= sketch.is_empty();
-  table_.theta_ = std::min(table_.theta_, sketch.get_theta64());
+  table_.theta_ = table_.is_empty_ ? theta_constants::MAX_THETA : std::min(table_.theta_, sketch.get_theta64());
   if (is_valid_ && table_.num_entries_ == 0) return;
   if (sketch.get_num_retained() == 0) {
     is_valid_ = true;
-    table_ = hash_table(0, 0, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+    table_ = hash_table(0, 0, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
     return;
   }
   if (!is_valid_) { // first update, copy or move incoming sketch
     is_valid_ = true;
     const uint8_t lg_size = lg_size_from_count(sketch.get_num_retained(), theta_update_sketch_base<EN, EK, A>::REBUILD_THRESHOLD);
-    table_ = hash_table(lg_size, lg_size, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+    table_ = hash_table(lg_size, lg_size, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
     for (auto& entry: sketch) {
       auto result = table_.find(EK()(entry));
       if (result.second) {
@@ -83,11 +83,11 @@ void theta_intersection_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
       throw std::invalid_argument(" fewer keys than expected, possibly corrupted input sketch");
     }
     if (match_count == 0) {
-      table_ = hash_table(0, 0, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+      table_ = hash_table(0, 0, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
       if (table_.theta_ == theta_constants::MAX_THETA) table_.is_empty_ = true;
     } else {
       const uint8_t lg_size = lg_size_from_count(match_count, theta_update_sketch_base<EN, EK, A>::REBUILD_THRESHOLD);
-      table_ = hash_table(lg_size, lg_size, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+      table_ = hash_table(lg_size, lg_size, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
       for (uint32_t i = 0; i < match_count; i++) {
         auto result = table_.find(EK()(matched_entries[i]));
         table_.insert(result.first, std::move(matched_entries[i]));

data/vendor/datasketches-cpp/theta/include/theta_set_difference_base_impl.hpp

@@ -36,7 +36,7 @@ seed_hash_(compute_seed_hash(seed))
 template<typename EN, typename EK, typename CS, typename A>
 template<typename FwdSketch, typename Sketch>
 CS theta_set_difference_base<EN, EK, CS, A>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const {
-  if (a.is_empty() || a.get_num_retained() == 0 || b.is_empty()) return CS(a, ordered);
+  if (a.is_empty() || (a.get_num_retained() > 0 && b.is_empty())) return CS(a, ordered);
   if (a.get_seed_hash() != seed_hash_) throw std::invalid_argument("A seed hash mismatch");
   if (b.get_seed_hash() != seed_hash_) throw std::invalid_argument("B seed hash mismatch");
 
@@ -53,7 +53,7 @@ CS theta_set_difference_base<EN, EK, CS, A>::compute(FwdSketch&& a, const Sketch
           conditional_back_inserter(entries, key_less_than<uint64_t, EN, EK>(theta)), comparator());
     } else { // hash-based
       const uint8_t lg_size = lg_size_from_count(b.get_num_retained(), hash_table::REBUILD_THRESHOLD);
-      hash_table table(lg_size, lg_size, hash_table::resize_factor::X1, 0, 0, allocator_); // theta and seed are not used here
+      hash_table table(lg_size, lg_size, hash_table::resize_factor::X1, 1, 0, 0, allocator_); // theta and seed are not used here
       for (const auto& entry: b) {
         const uint64_t hash = EK()(entry);
         if (hash < theta) {

data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp

@@ -25,14 +25,10 @@
 namespace datasketches {
 
 template<typename Allocator = std::allocator<uint64_t>>
-class theta_sketch_alloc {
+class base_theta_sketch_alloc {
 public:
-  using Entry = uint64_t;
-  using ExtractKey = trivial_extract_key;
-  using iterator = theta_iterator<Entry, ExtractKey>;
-  using const_iterator = theta_const_iterator<Entry, ExtractKey>;
 
-  virtual ~theta_sketch_alloc() = default;
+  virtual ~base_theta_sketch_alloc() = default;
 
   /**
    * @return allocator
@@ -104,6 +100,21 @@ public:
    */
   virtual string<Allocator> to_string(bool print_items = false) const;
 
+protected:
+  virtual void print_specifics(std::ostringstream& os) const = 0;
+  virtual void print_items(std::ostringstream& os) const = 0;
+};
+
+template<typename Allocator = std::allocator<uint64_t>>
+class theta_sketch_alloc: public base_theta_sketch_alloc<Allocator> {
+public:
+  using Entry = uint64_t;
+  using ExtractKey = trivial_extract_key;
+  using iterator = theta_iterator<Entry, ExtractKey>;
+  using const_iterator = theta_const_iterator<Entry, ExtractKey>;
+
+  virtual ~theta_sketch_alloc() = default;
+
   /**
    * Iterator over hash values in this sketch.
    * @return begin iterator
@@ -131,8 +142,7 @@ public:
   virtual const_iterator end() const = 0;
 
 protected:
-  using ostrstream = std::basic_ostringstream<char, std::char_traits<char>, AllocChar<Allocator>>;
-  virtual void print_specifics(ostrstream& os) const = 0;
+  virtual void print_items(std::ostringstream& os) const;
 };
 
 // forward declaration
@@ -269,6 +279,11 @@ public:
    */
   void trim();
 
+  /**
+   * Reset the sketch to the initial empty state
+   */
+  void reset();
+
   /**
    * Converts this sketch to a compact sketch (ordered or unordered).
    * @param ordered optional flag to specify if ordered sketch should be produced
@@ -285,11 +300,10 @@ private:
   theta_table table_;
 
   // for builder
-  update_theta_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta,
-      uint64_t seed, const Allocator& allocator);
+  update_theta_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p,
+      uint64_t theta, uint64_t seed, const Allocator& allocator);
 
-  using ostrstream = typename Base::ostrstream;
-  virtual void print_specifics(ostrstream& os) const;
+  virtual void print_specifics(std::ostringstream& os) const;
 };
 
 // compact sketch
@@ -377,8 +391,7 @@ private:
   uint64_t theta_;
   std::vector<uint64_t, Allocator> entries_;
 
-  using ostrstream = typename Base::ostrstream;
-  virtual void print_specifics(ostrstream& os) const;
+  virtual void print_specifics(std::ostringstream& os) const;
 };
 
 template<typename Allocator>
@@ -392,7 +405,7 @@ public:
 // It does not take the ownership of the buffer.
 
 template<typename Allocator = std::allocator<uint64_t>>
-class wrapped_compact_theta_sketch_alloc {
+class wrapped_compact_theta_sketch_alloc : public base_theta_sketch_alloc<Allocator> {
 public:
   using const_iterator = const uint64_t*;
 
@@ -415,6 +428,10 @@ public:
    */
   static const wrapped_compact_theta_sketch_alloc wrap(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED, bool dump_on_error = false);
 
+protected:
+  virtual void print_specifics(std::ostringstream& os) const;
+  virtual void print_items(std::ostringstream& os) const;
+
 private:
   bool is_empty_;
   bool is_ordered_;