datasketches 0.2.2 → 0.2.5

data/vendor/datasketches-cpp/sampling/include/var_opt_sketch.hpp

@@ -51,18 +51,27 @@ struct subset_summary {
   double total_sketch_weight;
 };
 
-enum resize_factor { X1 = 0, X2, X4, X8 };
-
 template <typename T, typename S, typename A> class var_opt_union; // forward declaration
 
-template <typename T, typename S = serde<T>, typename A = std::allocator<T>>
+namespace var_opt_constants {
+    const resize_factor DEFAULT_RESIZE_FACTOR = resize_factor::X8;
+    const uint32_t MAX_K = ((uint32_t) 1 << 31) - 2; 
+}
+
+template<
+  typename T,
+  typename S = serde<T>, // deprecated, to be removed in the next major version
+  typename A = std::allocator<T>
+>
 class var_opt_sketch {
 
   public:
-    static const resize_factor DEFAULT_RESIZE_FACTOR = X8;
-    static const uint32_t MAX_K = ((uint32_t) 1 << 31) - 2;
+    static const resize_factor DEFAULT_RESIZE_FACTOR = var_opt_constants::DEFAULT_RESIZE_FACTOR;
+    static const uint32_t MAX_K = var_opt_constants::MAX_K;
 
-    explicit var_opt_sketch(uint32_t k, resize_factor rf = DEFAULT_RESIZE_FACTOR, const A& allocator = A());
+    explicit var_opt_sketch(uint32_t k,
+      resize_factor rf = var_opt_constants::DEFAULT_RESIZE_FACTOR,
+      const A& allocator = A());
     var_opt_sketch(const var_opt_sketch& other);
     var_opt_sketch(var_opt_sketch&& other) noexcept;
 
@@ -130,18 +139,20 @@ class var_opt_sketch {
     /**
      * Computes size needed to serialize the current state of the sketch.
      * This version is for fixed-size arithmetic types (integral and floating point).
+     * @param instance of a SerDe
      * @return size in bytes needed to serialize this sketch
      */
-    template<typename TT = T, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type = 0>
-    inline size_t get_serialized_size_bytes() const;
+    template<typename TT = T, typename SerDe = S, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type = 0>
+    inline size_t get_serialized_size_bytes(const SerDe& sd = SerDe()) const;
 
     /**
      * Computes size needed to serialize the current state of the sketch.
      * This version is for all other types and can be expensive since every item needs to be looked at.
+     * @param instance of a SerDe
      * @return size in bytes needed to serialize this sketch
      */
-    template<typename TT = T, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type = 0>
-    inline size_t get_serialized_size_bytes() const;
+    template<typename TT = T, typename SerDe = S, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type = 0>
+    inline size_t get_serialized_size_bytes(const SerDe& sd = SerDe()) const;
 
     // This is a convenience alias for users
     // The type returned by the following serialize method
@@ -153,30 +164,61 @@ class var_opt_sketch {
      * It is a blank space of a given size.
      * This header is used in Datasketches PostgreSQL extension.
      * @param header_size_bytes space to reserve in front of the sketch
+     * @param instance of a SerDe
      */
-    vector_bytes serialize(unsigned header_size_bytes = 0) const;
+    template<typename SerDe = S>
+    vector_bytes serialize(unsigned header_size_bytes = 0, const SerDe& sd = SerDe()) const;
 
     /**
      * This method serializes the sketch into a given stream in a binary form
      * @param os output stream
+     * @param instance of a SerDe
      */
-    void serialize(std::ostream& os) const;
+    template<typename SerDe = S>
+    void serialize(std::ostream& os, const SerDe& sd = SerDe()) const;
 
     /**
      * This method deserializes a sketch from a given stream.
      * @param is input stream
+     * @param instance of an Allocator
      * @return an instance of a sketch
+     *
+     * Deprecated, to be removed in the next major version
      */
     static var_opt_sketch deserialize(std::istream& is, const A& allocator = A());
 
+    /**
+     * This method deserializes a sketch from a given stream.
+     * @param is input stream
+     * @param instance of a SerDe
+     * @param instance of an Allocator
+     * @return an instance of a sketch
+     */
+    template<typename SerDe = S>
+    static var_opt_sketch deserialize(std::istream& is, const SerDe& sd = SerDe(), const A& allocator = A());
+
     /**
      * This method deserializes a sketch from a given array of bytes.
      * @param bytes pointer to the array of bytes
      * @param size the size of the array
+     * @param instance of an Allocator
      * @return an instance of a sketch
+     *
+     * Deprecated, to be removed in the next major version
      */
     static var_opt_sketch deserialize(const void* bytes, size_t size, const A& allocator = A());
 
+    /**
+     * This method deserializes a sketch from a given array of bytes.
+     * @param bytes pointer to the array of bytes
+     * @param size the size of the array
+     * @param instance of a SerDe
+     * @param instance of an Allocator
+     * @return an instance of a sketch
+     */
+    template<typename SerDe = S>
+    static var_opt_sketch deserialize(const void* bytes, size_t size, const SerDe& sd = SerDe(), const A& allocator = A());
+
     /**
      * Prints a summary of the sketch.
      * @return the summary as a string

data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp

@@ -25,6 +25,7 @@
 #include <cmath>
 #include <random>
 #include <algorithm>
+#include <stdexcept>
 
 #include "var_opt_sketch.hpp"
 #include "serde.hpp"
@@ -128,7 +129,7 @@ var_opt_sketch<T,S,A>::var_opt_sketch(T* data, double* weights, size_t len,
   r_(r_count),
   n_(n),
   total_wt_r_(total_wt_r),
-  rf_(DEFAULT_RESIZE_FACTOR),
+  rf_(var_opt_constants::DEFAULT_RESIZE_FACTOR),
   curr_items_alloc_(len),
   filled_data_(n > k),
   allocator_(allocator),
@@ -311,8 +312,8 @@ var_opt_sketch<T,S,A>& var_opt_sketch<T,S,A>::operator=(var_opt_sketch&& other)
 
 // implementation for fixed-size arithmetic types (integral and floating point)
 template<typename T, typename S, typename A>
-template<typename TT, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type>
-size_t var_opt_sketch<T,S,A>::get_serialized_size_bytes() const {
+template<typename TT, typename SerDe, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type>
+size_t var_opt_sketch<T,S,A>::get_serialized_size_bytes(const SerDe&) const {
   if (is_empty()) { return PREAMBLE_LONGS_EMPTY << 3; }
   size_t num_bytes = (r_ == 0 ? PREAMBLE_LONGS_WARMUP : PREAMBLE_LONGS_FULL) << 3;
   num_bytes += h_ * sizeof(double);    // weights
@@ -325,8 +326,8 @@ size_t var_opt_sketch<T,S,A>::get_serialized_size_bytes() const {
 
 // implementation for all other types
 template<typename T, typename S, typename A>
-template<typename TT, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type>
-size_t var_opt_sketch<T,S,A>::get_serialized_size_bytes() const {
+template<typename TT, typename SerDe, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type>
+size_t var_opt_sketch<T,S,A>::get_serialized_size_bytes(const SerDe& sd) const {
   if (is_empty()) { return PREAMBLE_LONGS_EMPTY << 3; }
   size_t num_bytes = (r_ == 0 ? PREAMBLE_LONGS_WARMUP : PREAMBLE_LONGS_FULL) << 3;
   num_bytes += h_ * sizeof(double);    // weights
@@ -335,13 +336,14 @@ size_t var_opt_sketch<T,S,A>::get_serialized_size_bytes() const {
   }
   // must iterate over the items
   for (auto it: *this)
-    num_bytes += S().size_of_item(it.first);
+    num_bytes += sd.size_of_item(it.first);
   return num_bytes;
 }
 
 template<typename T, typename S, typename A>
-std::vector<uint8_t, AllocU8<A>> var_opt_sketch<T,S,A>::serialize(unsigned header_size_bytes) const {
-  const size_t size = header_size_bytes + get_serialized_size_bytes();
+template<typename SerDe>
+std::vector<uint8_t, AllocU8<A>> var_opt_sketch<T,S,A>::serialize(unsigned header_size_bytes, const SerDe& sd) const {
+  const size_t size = header_size_bytes + get_serialized_size_bytes(sd);
   std::vector<uint8_t, AllocU8<A>> bytes(size, 0, allocator_);
   uint8_t* ptr = bytes.data() + header_size_bytes;
   uint8_t* end_ptr = ptr + size;
@@ -400,8 +402,8 @@ std::vector<uint8_t, AllocU8<A>> var_opt_sketch<T,S,A>::serialize(unsigned heade
     }
 
     // write the sample items, skipping the gap. Either h_ or r_ may be 0
-    ptr += S().serialize(ptr, end_ptr - ptr, data_, h_);
-    ptr += S().serialize(ptr, end_ptr - ptr, &data_[h_ + 1], r_);
+    ptr += sd.serialize(ptr, end_ptr - ptr, data_, h_);
+    ptr += sd.serialize(ptr, end_ptr - ptr, &data_[h_ + 1], r_);
   }
   
   size_t bytes_written = ptr - bytes.data();
@@ -413,7 +415,8 @@ std::vector<uint8_t, AllocU8<A>> var_opt_sketch<T,S,A>::serialize(unsigned heade
 }
 
 template<typename T, typename S, typename A>
-void var_opt_sketch<T,S,A>::serialize(std::ostream& os) const {
+template<typename SerDe>
+void var_opt_sketch<T,S,A>::serialize(std::ostream& os, const SerDe& sd) const {
   const bool empty = (h_ == 0) && (r_ == 0);
 
   const uint8_t preLongs = (empty ? PREAMBLE_LONGS_EMPTY
@@ -469,13 +472,19 @@ void var_opt_sketch<T,S,A>::serialize(std::ostream& os) const {
     }
 
     // write the sample items, skipping the gap. Either h_ or r_ may be 0
-    S().serialize(os, data_, h_);
-    S().serialize(os, &data_[h_ + 1], r_);
+    sd.serialize(os, data_, h_);
+    sd.serialize(os, &data_[h_ + 1], r_);
   }
 }
 
 template<typename T, typename S, typename A>
 var_opt_sketch<T,S,A> var_opt_sketch<T,S,A>::deserialize(const void* bytes, size_t size, const A& allocator) {
+  return deserialize(bytes, size, S(), allocator);
+}
+
+template<typename T, typename S, typename A>
+template<typename SerDe>
+var_opt_sketch<T,S,A> var_opt_sketch<T,S,A>::deserialize(const void* bytes, size_t size, const SerDe& sd, const A& allocator) {
   ensure_minimum_memory(size, 8);
   const char* ptr = static_cast<const char*>(bytes);
   const char* base = ptr;
@@ -559,10 +568,10 @@ var_opt_sketch<T,S,A> var_opt_sketch<T,S,A>::deserialize(const void* bytes, size
   items_deleter deleter(array_size, allocator);
   std::unique_ptr<T, items_deleter> items(A(allocator).allocate(array_size), deleter);
   
-  ptr += S().deserialize(ptr, end_ptr - ptr, items.get(), h);
+  ptr += sd.deserialize(ptr, end_ptr - ptr, items.get(), h);
   items.get_deleter().set_h(h); // serde didn't throw, so the items are now valid
   
-  ptr += S().deserialize(ptr, end_ptr - ptr, &(items.get()[h + 1]), r);
+  ptr += sd.deserialize(ptr, end_ptr - ptr, &(items.get()[h + 1]), r);
   items.get_deleter().set_r(r); // serde didn't throw, so the items are now valid
 
   return var_opt_sketch(k, h, (r > 0 ? 1 : 0), r, n, total_wt_r, rf, array_size, false,
@@ -571,6 +580,12 @@ var_opt_sketch<T,S,A> var_opt_sketch<T,S,A>::deserialize(const void* bytes, size
 
 template<typename T, typename S, typename A>
 var_opt_sketch<T,S,A> var_opt_sketch<T,S,A>::deserialize(std::istream& is, const A& allocator) {
+  return deserialize(is, S(), allocator);
+}
+
+template<typename T, typename S, typename A>
+template<typename SerDe>
+var_opt_sketch<T,S,A> var_opt_sketch<T,S,A>::deserialize(std::istream& is, const SerDe& sd, const A& allocator) {
   const auto first_byte = read<uint8_t>(is);
   uint8_t preamble_longs = first_byte & 0x3f;
   const resize_factor rf = static_cast<resize_factor>((first_byte >> 6) & 0x03);
@@ -640,10 +655,10 @@ var_opt_sketch<T,S,A> var_opt_sketch<T,S,A>::deserialize(std::istream& is, const
   items_deleter deleter(array_size, allocator);
   std::unique_ptr<T, items_deleter> items(A(allocator).allocate(array_size), deleter);
 
-  S().deserialize(is, items.get(), h); // aka &data_[0]
+  sd.deserialize(is, items.get(), h); // aka &data_[0]
   items.get_deleter().set_h(h); // serde didn't throw, so the items are now valid
 
-  S().deserialize(is, &(items.get()[h + 1]), r);
+  sd.deserialize(is, &(items.get()[h + 1]), r);
   items.get_deleter().set_r(r); // serde didn't throw, so the items are now valid
 
   if (!is.good())
@@ -731,8 +746,10 @@ void var_opt_sketch<T,S,A>::update(T&& item, double weight) {
 
 template<typename T, typename S, typename A>
 string<A> var_opt_sketch<T,S,A>::to_string() const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
-  os << "### VarOpt SUMMARY: " << std::endl;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
+  os << "### VarOpt SUMMARY:" << std::endl;
   os << "   k            : " << k_ << std::endl;
   os << "   h            : " << h_ << std::endl;
   os << "   r            : " << r_ << std::endl;
@@ -740,24 +757,28 @@ string<A> var_opt_sketch<T,S,A>::to_string() const {
   os << "   Current size : " << curr_items_alloc_ << std::endl;
   os << "   Resize factor: " << (1 << rf_) << std::endl;
   os << "### END SKETCH SUMMARY" << std::endl;
-  return os.str();
+  return string<A>(os.str().c_str(), allocator_);
 }
 
 template<typename T, typename S, typename A>
 string<A> var_opt_sketch<T,S,A>::items_to_string() const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
   os << "### Sketch Items" << std::endl;
   int idx = 0;
   for (auto record : *this) {
     os << idx << ": " << record.first << "\twt = " << record.second << std::endl;
     ++idx;
   }
-  return os.str();
+  return string<A>(os.str().c_str(), allocator_);
 }
 
 template<typename T, typename S, typename A>
 string<A> var_opt_sketch<T,S,A>::items_to_string(bool print_gap) const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
   os << "### Sketch Items" << std::endl;
   const uint32_t array_length = (n_ < k_ ? n_ : k_ + 1);
   for (uint32_t i = 0, display_idx = 0; i < array_length; ++i) {
@@ -774,7 +795,7 @@ string<A> var_opt_sketch<T,S,A>::items_to_string(bool print_gap) const {
       ++display_idx;
     }
   }
-  return os.str();
+  return string<A>(os.str().c_str(), allocator_);
 }
 
 template<typename T, typename S, typename A>
@@ -1677,16 +1698,6 @@ bool var_opt_sketch<T, S, A>::iterator::get_mark() const {
   return sk_->marks_ == nullptr ? false : sk_->marks_[idx_];
 }
 
-
-
-// ******************** MOVE TO COMMON UTILS AREA EVENTUALLY *********************
-
-namespace random_utils {
-  static std::random_device rd; // possibly unsafe in MinGW with GCC < 9.2
-  static std::mt19937_64 rand(rd());
-  static std::uniform_real_distribution<> next_double(0.0, 1.0);
-}
-
 /**
  * Checks if target sampling allocation is more than 50% of max sampling size.
  * If so, returns max sampling size, otherwise passes through target size.

data/vendor/datasketches-cpp/sampling/include/var_opt_union.hpp

@@ -45,7 +45,11 @@ template<typename A> using AllocU8 = typename std::allocator_traits<A>::template
  * author Kevin Lang 
  * author Jon Malkin
  */
-template <typename T, typename S = serde<T>, typename A = std::allocator<T>>
+template<
+  typename T,
+  typename S = serde<T>, // deprecated, to be removed in the next major version
+  typename A = std::allocator<T>
+>
 class var_opt_union {
 
 public:
@@ -88,14 +92,16 @@ public:
   /**
    * Computes size needed to serialize the current state of the union.
    * This version is for all other types and can be expensive since every item needs to be looked at.
+   * @param instance of a SerDe
    * @return size in bytes needed to serialize this sketch
    */
-  size_t get_serialized_size_bytes() const;
-  
+  template<typename SerDe = S>
+  size_t get_serialized_size_bytes(const SerDe& sd = SerDe()) const;
+
   // This is a convenience alias for users
   // The type returned by the following serialize method
   typedef vector_u8<A> vector_bytes;
-  
+
   /**
    * NOTE: This method may be deprecated in a future version.
    * This method serializes the sketch as a vector of bytes.
@@ -103,33 +109,62 @@ public:
    * It is a blank space of a given size.
    * This header is used in Datasketches PostgreSQL extension.
    * @param header_size_bytes space to reserve in front of the sketch
+   * @param instance of a SerDe
    */
-  vector_bytes serialize(unsigned header_size_bytes = 0) const;
+  template<typename SerDe = S>
+  vector_bytes serialize(unsigned header_size_bytes = 0, const SerDe& sd = SerDe()) const;
 
   /**
    * NOTE: This method may be deprecated in a future version.
    * This method serializes the sketch into a given stream in a binary form
    * @param os output stream
+   * @param instance of a SerDe
    */
-  void serialize(std::ostream& os) const;
+  template<typename SerDe = S>
+  void serialize(std::ostream& os, const SerDe& sd = SerDe()) const;
 
   /**
    * NOTE: This method may be deprecated in a future version.
    * This method deserializes a union from a given stream.
    * @param is input stream
+   * @param instance of an Allocator
    * @return an instance of a union
    */
   static var_opt_union deserialize(std::istream& is, const A& allocator = A());
 
+  /**
+   * NOTE: This method may be deprecated in a future version.
+   * This method deserializes a union from a given stream.
+   * @param is input stream
+   * @param instance of a SerDe
+   * @param instance of an Allocator
+   * @return an instance of a union
+   */
+  template<typename SerDe = S>
+  static var_opt_union deserialize(std::istream& is, const SerDe& sd = SerDe(), const A& allocator = A());
+
   /**
    * NOTE: This method may be deprecated in a future version.
    * This method deserializes a union from a given array of bytes.
    * @param bytes pointer to the array of bytes
    * @param size the size of the array
+   * @param instance of an Allocator
    * @return an instance of a union
    */
   static var_opt_union deserialize(const void* bytes, size_t size, const A& allocator = A());
 
+  /**
+   * NOTE: This method may be deprecated in a future version.
+   * This method deserializes a union from a given array of bytes.
+   * @param bytes pointer to the array of bytes
+   * @param size the size of the array
+   * @param instance of a SerDe
+   * @param instance of an Allocator
+   * @return an instance of a union
+   */
+  template<typename SerDe = S>
+  static var_opt_union deserialize(const void* bytes, size_t size, const SerDe& sd = SerDe(), const A& allocator = A());
+
   /**
    * Prints a summary of the union as a string.
    * @return the summary as a string

data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp

@@ -24,6 +24,7 @@
 
 #include <cmath>
 #include <sstream>
+#include <stdexcept>
 
 namespace datasketches {
 
@@ -129,6 +130,12 @@ var_opt_union<T,S,A>& var_opt_union<T,S,A>::operator=(var_opt_union&& other) {
 
 template<typename T, typename S, typename A>
 var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(std::istream& is, const A& allocator) {
+  return deserialize(is, S(), allocator);
+}
+
+template<typename T, typename S, typename A>
+template<typename SerDe>
+var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(std::istream& is, const SerDe& sd, const A& allocator) {
   const auto preamble_longs = read<uint8_t>(is);
   const auto serial_version = read<uint8_t>(is);
   const auto family_id = read<uint8_t>(is);
@@ -155,7 +162,7 @@ var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(std::istream& is, const A
   const auto outer_tau_numer = read<double>(is);
   const auto outer_tau_denom = read<uint64_t>(is);
 
-  var_opt_sketch<T,S,A> gadget = var_opt_sketch<T,S,A>::deserialize(is, allocator);
+  var_opt_sketch<T,S,A> gadget = var_opt_sketch<T,S,A>::deserialize(is, sd, allocator);
 
   if (!is.good())
     throw std::runtime_error("error reading from std::istream"); 
@@ -165,6 +172,12 @@ var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(std::istream& is, const A
 
 template<typename T, typename S, typename A>
 var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(const void* bytes, size_t size, const A& allocator) {
+  return deserialize(bytes, size, S(), allocator);
+}
+
+template<typename T, typename S, typename A>
+template<typename SerDe>
+var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(const void* bytes, size_t size, const SerDe& sd, const A& allocator) {
   ensure_minimum_memory(size, 8);
   const char* ptr = static_cast<const char*>(bytes);
   uint8_t preamble_longs;
@@ -199,22 +212,24 @@ var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(const void* bytes, size_t
   ptr += copy_from_mem(ptr, outer_tau_denom);
 
   const size_t gadget_size = size - (PREAMBLE_LONGS_NON_EMPTY << 3);
-  var_opt_sketch<T,S,A> gadget = var_opt_sketch<T,S,A>::deserialize(ptr, gadget_size, allocator);
+  var_opt_sketch<T,S,A> gadget = var_opt_sketch<T,S,A>::deserialize(ptr, gadget_size, sd, allocator);
 
   return var_opt_union<T,S,A>(items_seen, outer_tau_numer, outer_tau_denom, max_k, std::move(gadget));
 }
 
 template<typename T, typename S, typename A>
-size_t var_opt_union<T,S,A>::get_serialized_size_bytes() const {
+template<typename SerDe>
+size_t var_opt_union<T,S,A>::get_serialized_size_bytes(const SerDe& sd) const {
   if (n_ == 0) {
     return PREAMBLE_LONGS_EMPTY << 3;
   } else {
-    return (PREAMBLE_LONGS_NON_EMPTY << 3) + gadget_.get_serialized_size_bytes();
+    return (PREAMBLE_LONGS_NON_EMPTY << 3) + gadget_.get_serialized_size_bytes(sd);
   }
 }
 
 template<typename T, typename S, typename A>
-void var_opt_union<T,S,A>::serialize(std::ostream& os) const {
+template<typename SerDe>
+void var_opt_union<T,S,A>::serialize(std::ostream& os, const SerDe& sd) const {
   bool empty = (n_ == 0);
 
   const uint8_t serialization_version(SER_VER);
@@ -240,13 +255,14 @@ void var_opt_union<T,S,A>::serialize(std::ostream& os) const {
     write(os, n_);
     write(os, outer_tau_numer_);
     write(os, outer_tau_denom_);
-    gadget_.serialize(os);
+    gadget_.serialize(os, sd);
   }
 }
 
 template<typename T, typename S, typename A>
-std::vector<uint8_t, AllocU8<A>> var_opt_union<T,S,A>::serialize(unsigned header_size_bytes) const {
-  const size_t size = header_size_bytes + get_serialized_size_bytes();
+template<typename SerDe>
+std::vector<uint8_t, AllocU8<A>> var_opt_union<T,S,A>::serialize(unsigned header_size_bytes, const SerDe& sd) const {
+  const size_t size = header_size_bytes + get_serialized_size_bytes(sd);
   std::vector<uint8_t, AllocU8<A>> bytes(size, 0, gadget_.allocator_);
   uint8_t* ptr = bytes.data() + header_size_bytes;
 
@@ -278,7 +294,7 @@ std::vector<uint8_t, AllocU8<A>> var_opt_union<T,S,A>::serialize(unsigned header
     ptr += copy_to_mem(outer_tau_numer_, ptr);
     ptr += copy_to_mem(outer_tau_denom_, ptr);
 
-    auto gadget_bytes = gadget_.serialize();
+    auto gadget_bytes = gadget_.serialize(0, sd);
     ptr += copy_to_mem(gadget_bytes.data(), ptr, gadget_bytes.size() * sizeof(uint8_t));
   }
 
@@ -295,14 +311,16 @@ void var_opt_union<T,S,A>::reset() {
 
 template<typename T, typename S, typename A>
 string<A> var_opt_union<T,S,A>::to_string() const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
-  os << "### VarOpt Union SUMMARY: " << std::endl;
-  os << " . n             : " << n_ << std::endl;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
+  os << "### VarOpt Union SUMMARY:" << std::endl;
+  os << "   n             : " << n_ << std::endl;
   os << "   Max k         : " << max_k_ << std::endl;
-  os << "   Gadget Summary: " << std::endl;
+  os << "   Gadget Summary:" << std::endl;
   os << gadget_.to_string();
-  os << "### END VarOpt Union SUMMARY: " << std::endl;
-  return os.str();
+  os << "### END VarOpt Union SUMMARY" << std::endl;
+  return string<A>(os.str().c_str(), gadget_.allocator_);
 }
 
 template<typename T, typename S, typename A>

data/vendor/datasketches-cpp/sampling/test/var_opt_allocation_test.cpp

@@ -39,7 +39,7 @@ TEST_CASE("varopt allocation test", "[var_opt_sketch]") {
     var_opt_test_sketch sk1(10, var_opt_test_sketch::DEFAULT_RESIZE_FACTOR, 0);
     for (int i = 0; i < 100; ++i) sk1.update(i);
     auto bytes1 = sk1.serialize();
-    auto sk2 = var_opt_test_sketch::deserialize(bytes1.data(), bytes1.size(), 0);
+    auto sk2 = var_opt_test_sketch::deserialize(bytes1.data(), bytes1.size(), test_type_serde(), 0);
 
     std::stringstream ss;
     sk1.serialize(ss);
@@ -51,7 +51,7 @@ TEST_CASE("varopt allocation test", "[var_opt_sketch]") {
     u1.update(sk3);
 
     auto bytes2 = u1.serialize();
-    auto u2 = var_opt_test_union::deserialize(bytes2.data(), bytes2.size(), 0);
+    auto u2 = var_opt_test_union::deserialize(bytes2.data(), bytes2.size(), test_type_serde(), 0);
   }
   REQUIRE(test_allocator_total_bytes == 0);
   REQUIRE(test_allocator_net_allocations == 0);

data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp

@@ -27,6 +27,7 @@
 #include <fstream>
 #include <cmath>
 #include <random>
+#include <stdexcept>
 
 #ifdef TEST_BINARY_INPUT_PATH
 static std::string testBinaryInputPath = TEST_BINARY_INPUT_PATH;

data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp

@@ -27,6 +27,7 @@
 #include <fstream>
 #include <cmath>
 #include <random>
+#include <stdexcept>
 
 #ifdef TEST_BINARY_INPUT_PATH
 static std::string testBinaryInputPath = TEST_BINARY_INPUT_PATH;

data/vendor/datasketches-cpp/setup.py

@@ -49,8 +49,9 @@ class CMakeBuild(build_ext):
             os.path.dirname(self.get_ext_fullpath(ext.name)))
         cmake_args =  ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir]
         cmake_args += ['-DWITH_PYTHON=True']
+        cmake_args += ['-DCMAKE_CXX_STANDARD=11']
         # ensure we use a consistent python version
-        cmake_args += ['-DPYTHON_EXECUTABLE=' + sys.executable]
+        cmake_args += ['-DPython3_EXECUTABLE=' + sys.executable]
         cfg = 'Debug' if self.debug else 'Release'
         build_args = ['--config', cfg]
 
@@ -59,7 +60,8 @@ class CMakeBuild(build_ext):
                            cfg.upper(),
                            extdir)]
             if sys.maxsize > 2**32:
-                cmake_args += ['-A', 'x64']
+                cmake_args += ['-T', 'host=x64']
+                cmake_args += ['-DCMAKE_GENERATOR_PLATFORM=x64']
                 build_args += ['--', '/m']
         else:
             cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
@@ -74,23 +76,24 @@ class CMakeBuild(build_ext):
         subprocess.check_call(['cmake', ext.sourcedir] + cmake_args,
                               cwd=self.build_temp, env=env)
         subprocess.check_call(['cmake', '--build', '.', '--target', 'python'] + build_args,
-                              cwd=self.build_temp)
+                              cwd=self.build_temp, env=env)
         print() # add an empty line to pretty print
 
 setup(
     name='datasketches',
-    version='3.1.0',
-    author='Apache DataSketches Developers',
+    version='3.4.0',
+    author='Apache Software Foundation',
     author_email='dev@datasketches.apache.org',
-    description='A wrapper for the C++ Apache DataSketches library',
+    description='The Apache DataSketches Library for Python',
     license='Apache License 2.0',
     url='http://datasketches.apache.org',
     long_description=open('python/README.md').read(),
+    long_description_content_type='text/markdown',
     packages=find_packages('python'), # python pacakges only in this dir
     package_dir={'':'python'},
     # may need to add all source paths for sdist packages w/o MANIFEST.in
     ext_modules=[CMakeExtension('datasketches')],
     cmdclass={'build_ext': CMakeBuild},
-    setup_requires=['setuptools_scm','tox-setuptools'],
+    install_requires=['numpy'],
     zip_safe=False
 )