datasketches 0.3.2 → 0.4.0

data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_serialize_for_java.cpp

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <frequent_items_sketch.hpp>
+
+namespace datasketches {
+
+TEST_CASE("frequent longs sketch generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    frequent_items_sketch<long> sketch(6);
+    for (unsigned i = 1; i <= n; ++i) sketch.update(i);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    if (n > 10) {
+      REQUIRE(sketch.get_maximum_error() > 0);
+    } else {
+      REQUIRE(sketch.get_maximum_error() == 0);
+    }
+    REQUIRE(sketch.get_total_weight() == n);
+    std::ofstream os("frequent_long_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.serialize(os);
+  }
+}
+
+TEST_CASE("frequent strings sketch generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    frequent_items_sketch<std::string> sketch(6);
+    for (unsigned i = 1; i <= n; ++i) sketch.update(std::to_string(i));
+    REQUIRE(sketch.is_empty() == (n == 0));
+    if (n > 10) {
+      REQUIRE(sketch.get_maximum_error() > 0);
+    } else {
+      REQUIRE(sketch.get_maximum_error() == 0);
+    }
+    REQUIRE(sketch.get_total_weight() == n);
+    std::ofstream os("frequent_string_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.serialize(os);
+  }
+}
+
+TEST_CASE("frequent strings sketch ascii", "[serialize_for_java]") {
+  frequent_items_sketch<std::string> sketch(6);
+  sketch.update("aaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 1);
+  sketch.update("bbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 2);
+  sketch.update("ccccccccccccccccccccccccccccc", 3);
+  sketch.update("ddddddddddddddddddddddddddddd", 4);
+  std::ofstream os("frequent_string_ascii_cpp.sk", std::ios::binary);
+  sketch.serialize(os);
+}
+
+TEST_CASE("frequent strings sketch utf8", "[serialize_for_java]") {
+  frequent_items_sketch<std::string> sketch(6);
+  sketch.update("абвгд", 1);
+  sketch.update("еёжзи", 2);
+  sketch.update("йклмн", 3);
+  sketch.update("опрст", 4);
+  sketch.update("уфхцч", 5);
+  sketch.update("шщъыь", 6);
+  sketch.update("эюя", 7);
+  std::ofstream os("frequent_string_utf8_cpp.sk", std::ios::binary);
+  sketch.serialize(os);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_test.cpp

@@ -70,6 +70,7 @@ TEST_CASE("frequent items: several items, no resize, no purge", "[frequent_items
   REQUIRE(sketch.get_estimate("b") == 3);
   REQUIRE(sketch.get_estimate("c") == 2);
   REQUIRE(sketch.get_estimate("d") == 1);
+  REQUIRE(sketch.get_maximum_error() == 0);
 }
 
 TEST_CASE("frequent items: several items, with resize, no purge", "[frequent_items_sketch]") {
@@ -96,6 +97,7 @@ TEST_CASE("frequent items: several items, with resize, no purge", "[frequent_ite
   REQUIRE(sketch.get_estimate("b") == 3);
   REQUIRE(sketch.get_estimate("c") == 2);
   REQUIRE(sketch.get_estimate("d") == 1);
+  REQUIRE(sketch.get_maximum_error() == 0);
 }
 
 TEST_CASE("frequent items: estimation mode", "[frequent_items_sketch]") {
@@ -149,6 +151,7 @@ TEST_CASE("frequent items: merge exact mode", "[frequent_items_sketch]") {
   REQUIRE(sketch1.get_estimate(2) == 3);
   REQUIRE(sketch1.get_estimate(3) == 2);
   REQUIRE(sketch1.get_estimate(4) == 1);
+  REQUIRE(sketch1.get_maximum_error() == 0);
 }
 
 TEST_CASE("frequent items: merge estimation mode", "[frequent_items_sketch]") {
@@ -199,48 +202,6 @@ TEST_CASE("frequent items: merge estimation mode", "[frequent_items_sketch]") {
   REQUIRE(9 <= items[1].get_estimate()); // always overestimated
 }
 
-TEST_CASE("frequent items: deserialize from java long", "[frequent_items_sketch]") {
-  std::ifstream is;
-  is.exceptions(std::ios::failbit | std::ios::badbit);
-  is.open(testBinaryInputPath + "longs_sketch_from_java.sk", std::ios::binary);
-  auto sketch = frequent_items_sketch<long long>::deserialize(is);
-  REQUIRE_FALSE(sketch.is_empty());
-  REQUIRE(sketch.get_total_weight() == 4);
-  REQUIRE(sketch.get_num_active_items() == 4);
-  REQUIRE(sketch.get_estimate(1) == 1);
-  REQUIRE(sketch.get_estimate(2) == 1);
-  REQUIRE(sketch.get_estimate(3) == 1);
-  REQUIRE(sketch.get_estimate(4) == 1);
-}
-
-TEST_CASE("frequent items: deserialize from java string", "[frequent_items_sketch]") {
-  std::ifstream is;
-  is.exceptions(std::ios::failbit | std::ios::badbit);
-  is.open(testBinaryInputPath + "items_sketch_string_from_java.sk", std::ios::binary);
-  auto sketch = frequent_items_sketch<std::string>::deserialize(is);
-  REQUIRE_FALSE(sketch.is_empty());
-  REQUIRE(sketch.get_total_weight() == 4);
-  REQUIRE(sketch.get_num_active_items() == 4);
-  REQUIRE(sketch.get_estimate("aaaaaaaaaaaaaaaaaaaaaaaaaaaaa") == 1);
-  REQUIRE(sketch.get_estimate("bbbbbbbbbbbbbbbbbbbbbbbbbbbbb") == 1);
-  REQUIRE(sketch.get_estimate("ccccccccccccccccccccccccccccc") == 1);
-  REQUIRE(sketch.get_estimate("ddddddddddddddddddddddddddddd") == 1);
-}
-
-TEST_CASE("frequent items: deserialize from java string, utf-8", "[frequent_items_sketch]") {
-  std::ifstream is;
-  is.exceptions(std::ios::failbit | std::ios::badbit);
-  is.open(testBinaryInputPath + "items_sketch_string_utf8_from_java.sk", std::ios::binary);
-  auto sketch = frequent_items_sketch<std::string>::deserialize(is);
-  REQUIRE_FALSE(sketch.is_empty());
-  REQUIRE(sketch.get_total_weight() == 10);
-  REQUIRE(sketch.get_num_active_items() == 4);
-  REQUIRE(sketch.get_estimate("абвгд") == 1);
-  REQUIRE(sketch.get_estimate("еёжзи") == 2);
-  REQUIRE(sketch.get_estimate("йклмн") == 3);
-  REQUIRE(sketch.get_estimate("опрст") == 4);
-}
-
 TEST_CASE("frequent items: deserialize long64 stream", "[frequent_items_sketch]") {
   frequent_items_sketch<long long> sketch1(3);
   sketch1.update(1, 1);

data/vendor/datasketches-cpp/hll/include/CouponList-internal.hpp

@@ -169,9 +169,9 @@ CouponList<A>* CouponList<A>::newList(std::istream& is, const A& allocator) {
 }
 
 template<typename A>
-vector_u8<A> CouponList<A>::serialize(bool compact, unsigned header_size_bytes) const {
+auto CouponList<A>::serialize(bool compact, unsigned header_size_bytes) const -> vector_bytes {
   const size_t sketchSizeBytes = (compact ? getCompactSerializationBytes() : getUpdatableSerializationBytes()) + header_size_bytes;
-  vector_u8<A> byteArr(sketchSizeBytes, 0, getAllocator());
+  vector_bytes byteArr(sketchSizeBytes, 0, getAllocator());
   uint8_t* bytes = byteArr.data() + header_size_bytes;
 
   bytes[hll_constants::PREAMBLE_INTS_BYTE] = static_cast<uint8_t>(getPreInts());

data/vendor/datasketches-cpp/hll/include/CouponList.hpp

@@ -33,12 +33,14 @@ class HllSketchImplFactory;
 template<typename A>
 class CouponList : public HllSketchImpl<A> {
   public:
+    using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
+
     CouponList(uint8_t lgConfigK, target_hll_type tgtHllType, hll_mode mode, const A& allocator);
     CouponList(const CouponList& that, target_hll_type tgtHllType);
 
     static CouponList* newList(const void* bytes, size_t len, const A& allocator);
     static CouponList* newList(std::istream& is, const A& allocator);
-    virtual vector_u8<A> serialize(bool compact, unsigned header_size_bytes) const;
+    virtual vector_bytes serialize(bool compact, unsigned header_size_bytes) const;
     virtual void serialize(std::ostream& os, bool compact) const;
 
     virtual ~CouponList() = default;

data/vendor/datasketches-cpp/hll/include/HllArray-internal.hpp

@@ -216,9 +216,9 @@ HllArray<A>* HllArray<A>::newHll(std::istream& is, const A& allocator) {
 }
 
 template<typename A>
-vector_u8<A> HllArray<A>::serialize(bool compact, unsigned header_size_bytes) const {
+auto HllArray<A>::serialize(bool compact, unsigned header_size_bytes) const -> vector_bytes {
   const size_t sketchSizeBytes = (compact ? getCompactSerializationBytes() : getUpdatableSerializationBytes()) + header_size_bytes;
-  vector_u8<A> byteArr(sketchSizeBytes, 0, getAllocator());
+  vector_bytes byteArr(sketchSizeBytes, 0, getAllocator());
   uint8_t* bytes = byteArr.data() + header_size_bytes;
   AuxHashMap<A>* auxHashMap = getAuxHashMap();
 
@@ -537,7 +537,7 @@ AuxHashMap<A>* HllArray<A>::getAuxHashMap() const {
 }
 
 template<typename A>
-const vector_u8<A>& HllArray<A>::getHllArray() const {
+auto HllArray<A>::getHllArray() const -> const vector_bytes& {
   return hllByteArr_;
 }
 

data/vendor/datasketches-cpp/hll/include/HllArray.hpp

@@ -31,13 +31,15 @@ class AuxHashMap;
 template<typename A>
 class HllArray : public HllSketchImpl<A> {
   public:
+    using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
+
     HllArray(uint8_t lgConfigK, target_hll_type tgtHllType, bool startFullSize, const A& allocator);
     explicit HllArray(const HllArray& other, target_hll_type tgtHllType);
 
     static HllArray* newHll(const void* bytes, size_t len, const A& allocator);
     static HllArray* newHll(std::istream& is, const A& allocator);
 
-    virtual vector_u8<A> serialize(bool compact, unsigned header_size_bytes) const;
+    virtual vector_bytes serialize(bool compact, unsigned header_size_bytes) const;
     virtual void serialize(std::ostream& os, bool compact) const;
 
     virtual ~HllArray() = default;
@@ -97,7 +99,7 @@ class HllArray : public HllSketchImpl<A> {
 
     virtual A getAllocator() const;
 
-    const vector_u8<A>& getHllArray() const;
+    const vector_bytes& getHllArray() const;
 
   protected:
     void hipAndKxQIncrementalUpdate(uint8_t oldValue, uint8_t newValue);
@@ -107,7 +109,7 @@ class HllArray : public HllSketchImpl<A> {
     double hipAccum_;
     double kxq0_;
     double kxq1_;
-    vector_u8<A> hllByteArr_; //init by sub-classes
+    vector_bytes hllByteArr_; //init by sub-classes
     uint8_t curMin_; //always zero for Hll6 and Hll8, only tracked by Hll4Array
     uint32_t numAtCurMin_; //interpreted as num zeros when curMin == 0
     bool oooFlag_; //Out-Of-Order Flag

data/vendor/datasketches-cpp/hll/include/HllSketch-internal.hpp

@@ -94,14 +94,14 @@ hll_sketch_alloc<A>::hll_sketch_alloc(HllSketchImpl<A>* that) :
 {}
 
 template<typename A>
-hll_sketch_alloc<A> hll_sketch_alloc<A>::operator=(const hll_sketch_alloc<A>& other) {
+hll_sketch_alloc<A>& hll_sketch_alloc<A>::operator=(const hll_sketch_alloc<A>& other) {
   sketch_impl->get_deleter()(sketch_impl);
   sketch_impl = other.sketch_impl->copy();
   return *this;
 }
 
 template<typename A>
-hll_sketch_alloc<A> hll_sketch_alloc<A>::operator=(hll_sketch_alloc<A>&& other) {
+hll_sketch_alloc<A>& hll_sketch_alloc<A>::operator=(hll_sketch_alloc<A>&& other) {
   std::swap(sketch_impl, other.sketch_impl);
   return *this;
 }
@@ -232,12 +232,12 @@ void hll_sketch_alloc<A>::serialize_updatable(std::ostream& os) const {
 }
 
 template<typename A>
-vector_u8<A> hll_sketch_alloc<A>::serialize_compact(unsigned header_size_bytes) const {
+auto hll_sketch_alloc<A>::serialize_compact(unsigned header_size_bytes) const -> vector_bytes {
   return sketch_impl->serialize(true, header_size_bytes);
 }
 
 template<typename A>
-vector_u8<A> hll_sketch_alloc<A>::serialize_updatable() const {
+auto hll_sketch_alloc<A>::serialize_updatable() const -> vector_bytes {
   return sketch_impl->serialize(false, 0);
 }
 

data/vendor/datasketches-cpp/hll/include/HllSketchImpl.hpp

@@ -30,11 +30,13 @@ namespace datasketches {
 template<typename A>
 class HllSketchImpl {
   public:
+    using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
+
     HllSketchImpl(uint8_t lgConfigK, target_hll_type tgtHllType, hll_mode mode, bool startFullSize);
     virtual ~HllSketchImpl();
 
     virtual void serialize(std::ostream& os, bool compact) const = 0;
-    virtual vector_u8<A> serialize(bool compact, unsigned header_size_bytes) const = 0;
+    virtual vector_bytes serialize(bool compact, unsigned header_size_bytes) const = 0;
 
     virtual HllSketchImpl* copy() const = 0;
     virtual HllSketchImpl* copyAs(target_hll_type tgtHllType) const = 0;

data/vendor/datasketches-cpp/hll/include/HllUtil.hpp

@@ -124,8 +124,6 @@ public:
   static uint32_t pair(uint32_t slotNo, uint8_t value);
   static uint32_t getLow26(uint32_t coupon);
   static uint8_t getValue(uint32_t coupon);
-  static double invPow2(uint8_t e);
-  static uint8_t ceilingPowerOf2(uint32_t n);
   static uint8_t simpleIntLog2(uint32_t n); // n must be power of 2
   static uint8_t computeLgArrInts(hll_mode mode, uint32_t count, uint8_t lgConfigK);
   static double getRelErr(bool upperBound, bool unioned, uint8_t lgConfigK, uint8_t numStdDev);
@@ -204,16 +202,6 @@ inline uint8_t HllUtil<A>::getValue(uint32_t coupon) {
   return coupon >> hll_constants::KEY_BITS_26;
 }
 
-template<typename A>
-inline double HllUtil<A>::invPow2(uint8_t e) {
-  union {
-    long long longVal;
-    double doubleVal;
-  } conv;
-  conv.longVal = (1023L - e) << 52;
-  return conv.doubleVal;
-}
-
 template<typename A>
 inline uint8_t HllUtil<A>::simpleIntLog2(uint32_t n) {
   if (n == 0) {

data/vendor/datasketches-cpp/hll/include/hll.hpp

@@ -30,40 +30,15 @@
 
 namespace datasketches {
 
-  /**
- * This is a high performance implementation of Phillipe Flajolet’s HLL sketch but with
- * significantly improved error behavior.  If the ONLY use case for sketching is counting
- * uniques and merging, the HLL sketch is a reasonable choice, although the highest performing in terms of accuracy for
- * storage space consumed is CPC (Compressed Probabilistic Counting). For large enough counts, this HLL version (with HLL_4) can be 2 to
- * 16 times smaller than the Theta sketch family for the same accuracy.
- *
- * <p>This implementation offers three different types of HLL sketch, each with different
- * trade-offs with accuracy, space and performance. These types are specified with the
- * {@link TgtHllType} parameter.
- *
- * <p>In terms of accuracy, all three types, for the same <i>lg_config_k</i>, have the same error
- * distribution as a function of <i>n</i>, the number of unique values fed to the sketch.
- * The configuration parameter <i>lg_config_k</i> is the log-base-2 of <i>K</i>,
- * where <i>K</i> is the number of buckets or slots for the sketch.
- *
- * <p>During warmup, when the sketch has only received a small number of unique items
- * (up to about 10% of <i>K</i>), this implementation leverages a new class of estimator
- * algorithms with significantly better accuracy.
- *
- * <p>This sketch also offers the capability of operating off-heap. Given a WritableMemory object
- * created by the user, the sketch will perform all of its updates and internal phase transitions
- * in that object, which can actually reside either on-heap or off-heap based on how it is
- * configured. In large systems that must update and merge many millions of sketches, having the
- * sketch operate off-heap avoids the serialization and deserialization costs of moving sketches
- * to and from off-heap memory-mapped files, for example, and eliminates big garbage collection
- * delays.
- *
- * author Jon Malkin
- * author Lee Rhodes
- * author Kevin Lang
- */
+// forward declarations
+template<typename A> class hll_sketch_alloc;
+template<typename A> class hll_union_alloc;
+
+/// HLL sketch alias with default allocator
+using hll_sketch = hll_sketch_alloc<std::allocator<uint8_t>>;
+/// HLL union alias with default allocator
+using hll_union = hll_union_alloc<std::allocator<uint8_t>>;
 
-  
 /**
  * Specifies the target type of HLL sketch to be created. It is a target in that the actual
  * allocation of the HLL array is deferred until sufficient number of items have been received by
@@ -100,14 +75,41 @@ enum target_hll_type {
     HLL_8  ///< 8 bits per entry (fastest, fixed size)
 };
 
-template<typename A>
-class HllSketchImpl;
-
-template<typename A>
-class hll_union_alloc;
+/**
+ * This is a high performance implementation of Phillipe Flajolet's HLL sketch but with
+ * significantly improved error behavior.  If the ONLY use case for sketching is counting
+ * uniques and merging, the HLL sketch is a reasonable choice, although the highest performing in terms of accuracy for
+ * storage space consumed is CPC (Compressed Probabilistic Counting). For large enough counts, this HLL version (with HLL_4) can be 2 to
+ * 16 times smaller than the Theta sketch family for the same accuracy.
+ *
+ * <p>This implementation offers three different types of HLL sketch, each with different
+ * trade-offs with accuracy, space and performance. These types are specified with the
+ * {@link target_hll_type} parameter.
+ *
+ * <p>In terms of accuracy, all three types, for the same <i>lg_config_k</i>, have the same error
+ * distribution as a function of <i>n</i>, the number of unique values fed to the sketch.
+ * The configuration parameter <i>lg_config_k</i> is the log-base-2 of <i>K</i>,
+ * where <i>K</i> is the number of buckets or slots for the sketch.
+ *
+ * <p>During warmup, when the sketch has only received a small number of unique items
+ * (up to about 10% of <i>K</i>), this implementation leverages a new class of estimator
+ * algorithms with significantly better accuracy.
+ *
+ * <p>This sketch also offers the capability of operating off-heap. Given a WritableMemory object
+ * created by the user, the sketch will perform all of its updates and internal phase transitions
+ * in that object, which can actually reside either on-heap or off-heap based on how it is
+ * configured. In large systems that must update and merge many millions of sketches, having the
+ * sketch operate off-heap avoids the serialization and deserialization costs of moving sketches
+ * to and from off-heap memory-mapped files, for example, and eliminates big garbage collection
+ * delays.
+ *
+ * author Jon Malkin
+ * author Lee Rhodes
+ * author Kevin Lang
+ */
 
-template<typename A> using AllocU8 = typename std::allocator_traits<A>::template rebind_alloc<uint8_t>;
-template<typename A> using vector_u8 = std::vector<uint8_t, AllocU8<A>>;
+// forward declaration
+template<typename A> class HllSketchImpl;
 
 template<typename A = std::allocator<uint8_t> >
 class hll_sketch_alloc final {
@@ -119,27 +121,33 @@ class hll_sketch_alloc final {
      * @param start_full_size Indicates whether to start in HLL mode,
      *        keeping memory use constant (if HLL_6 or HLL_8) at the cost of
      *        starting out using much more memory
+     * @param allocator instance of an Allocator
      */
     explicit hll_sketch_alloc(uint8_t lg_config_k, target_hll_type tgt_type = HLL_4, bool start_full_size = false, const A& allocator = A());
 
     /**
      * Copy constructor
+     * @param that sketch to be copied
      */
     hll_sketch_alloc(const hll_sketch_alloc<A>& that);
 
     /**
      * Copy constructor to a new target type
+     * @param that sketch to be copied
+     * @param tgt_type target_hll_type
      */
     hll_sketch_alloc(const hll_sketch_alloc<A>& that, target_hll_type tgt_type);
 
     /**
      * Move constructor
+     * @param that sketch to be moved
      */
     hll_sketch_alloc(hll_sketch_alloc<A>&& that) noexcept;
 
     /**
      * Reconstructs a sketch from a serialized image on a stream.
      * @param is An input stream with a binary image of a sketch
+     * @param allocator instance of an Allocator
      */
     static hll_sketch_alloc deserialize(std::istream& is, const A& allocator = A());
 
@@ -147,17 +155,26 @@ class hll_sketch_alloc final {
      * Reconstructs a sketch from a serialized image in a byte array.
      * @param bytes An input array with a binary image of a sketch
      * @param len Length of the input array, in bytes
+     * @param allocator instance of an Allocator
      */
     static hll_sketch_alloc deserialize(const void* bytes, size_t len, const A& allocator = A());
 
     //! Class destructor
     virtual ~hll_sketch_alloc();
 
-    //! Copy assignment operator
-    hll_sketch_alloc operator=(const hll_sketch_alloc<A>& other);
+    /**
+     *  Copy assignment operator
+     *  @param other sketch to be copied
+     *  @return reference to this sketch
+     */
+    hll_sketch_alloc& operator=(const hll_sketch_alloc<A>& other);
 
-    //! Move assignment operator
-    hll_sketch_alloc operator=(hll_sketch_alloc<A>&& other);
+    /**
+     * Move assignment operator
+     * @param other sketch to be moved
+     * @return reference to this sketch
+     */
+    hll_sketch_alloc& operator=(hll_sketch_alloc<A>&& other);
 
     /**
      * Resets the sketch to an empty state in coupon collection mode.
@@ -165,18 +182,22 @@ class hll_sketch_alloc final {
      */
     void reset();
 
-    typedef vector_u8<A> vector_bytes; // alias for users
+    // This is a convenience alias for users
+    // The type returned by the following serialize method
+    using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
 
     /**
      * Serializes the sketch to a byte array, compacting data structures
      * where feasible to eliminate unused storage in the serialized image.
      * @param header_size_bytes Allows for PostgreSQL integration
+     * @return serialized sketch in binary form
      */
     vector_bytes serialize_compact(unsigned header_size_bytes = 0) const;
 
     /**
      * Serializes the sketch to a byte array, retaining all internal 
      * data structures in their current form.
+     * @return serialized sketch in binary form
      */
     vector_bytes serialize_updatable() const;
 
@@ -392,8 +413,6 @@ class hll_sketch_alloc final {
     bool is_out_of_order_flag() const;
     bool is_estimation_mode() const;
 
-    typedef typename std::allocator_traits<A>::template rebind_alloc<hll_sketch_alloc> AllocHllSketch;
-
     HllSketchImpl<A>* sketch_impl;
     friend hll_union_alloc<A>;
 };
@@ -413,8 +432,8 @@ class hll_sketch_alloc final {
  * <p>Although the API for this union operator parallels many of the methods of the
  * <i>HllSketch</i>, the behavior of the union operator has some fundamental differences.
  *
- * <p>First, the user cannot specify the #tgt_hll_type as an input parameter.
- * Instead, it is specified for the sketch returned with #get_result(tgt_hll_tyope).
+ * <p>First, the user cannot specify the #target_hll_type as an input parameter.
+ * Instead, it is specified for the sketch returned with #get_result.
  *
  * <p>Second, the internal effective value of log-base-2 of <i>k</i> for the union operation can
  * change dynamically based on the smallest <i>lg_config_k</i> that the union operation has seen.
@@ -423,7 +442,6 @@ class hll_sketch_alloc final {
  * author Lee Rhodes
  * author Kevin Lang
  */
- 
 template<typename A = std::allocator<uint8_t> >
 class hll_union_alloc {
   public:
@@ -431,6 +449,7 @@ class hll_union_alloc {
      * Construct an hll_union operator with the given maximum log2 of k.
      * @param lg_max_k The maximum size, in log2, of k. The value must
      * be between 7 and 21, inclusive.
+     * @param allocator instance of an Allocator
      */
     explicit hll_union_alloc(uint8_t lg_max_k, const A& allocator = A());
 
@@ -495,7 +514,7 @@ class hll_union_alloc {
 
     /**
      * Returns the result of this union operator with the specified
-     * #tgt_hll_type.
+     * #target_hll_type.
      * @param tgt_type The tgt_hll_type enum value of the desired result (Default: HLL_4)
      * @return The result of this union with the specified tgt_hll_type
      */
@@ -629,12 +648,6 @@ class hll_union_alloc {
     hll_sketch_alloc<A> gadget_;
 };
 
-/// convenience alias for hll_sketch with default allocator
-typedef hll_sketch_alloc<> hll_sketch;
-
-/// convenience alias for hll_union with default allocator
-typedef hll_union_alloc<> hll_union;
-
 } // namespace datasketches
 
 #include "hll.private.hpp"

data/vendor/datasketches-cpp/hll/test/CMakeLists.txt

@@ -20,7 +20,6 @@ add_executable(hll_test)
 target_link_libraries(hll_test hll common_test_lib)
 
 set_target_properties(hll_test PROPERTIES
-  CXX_STANDARD 11
   CXX_STANDARD_REQUIRED YES
 )
 
@@ -49,3 +48,17 @@ target_sources(hll_test
     ToFromByteArrayTest.cpp
     IsomorphicTest.cpp
 )
+
+if (SERDE_COMPAT)
+target_sources(hll_test
+  PRIVATE
+   hll_sketch_deserialize_from_java_test.cpp
+)
+endif()
+
+if (GENERATE)
+target_sources(hll_test
+  PRIVATE
+   hll_sketch_serialize_for_java.cpp
+)
+endif()

data/vendor/datasketches-cpp/hll/test/ToFromByteArrayTest.cpp

@@ -53,74 +53,6 @@ TEST_CASE("hll to/from byte array: double serialize", "[hll_byte_array]") {
   }
 }
 
-TEST_CASE("hll to/from byte array: deserialize from java", "[hll_byte_array]") {
-  std::string inputPath;
-#ifdef TEST_BINARY_INPUT_PATH
-    inputPath = TEST_BINARY_INPUT_PATH;
-#else
-    inputPath = "test/";
-#endif
-
-  std::ifstream ifs;
-  ifs.open(inputPath + "list_from_java.sk", std::ios::binary);
-  hll_sketch sk = hll_sketch::deserialize(ifs);
-  REQUIRE(sk.is_empty() == false);
-  REQUIRE(sk.get_lg_config_k() == 8);
-  REQUIRE(sk.get_lower_bound(1) == 7.0);
-  REQUIRE(sk.get_estimate() == Approx(7.0).margin(1e-6));
-  REQUIRE(sk.get_upper_bound(1) == Approx(7.000350).margin(1e-5));
-  ifs.close();
-
-  ifs.open(inputPath + "compact_set_from_java.sk", std::ios::binary);
-  sk = hll_sketch::deserialize(ifs);
-  REQUIRE(sk.is_empty() == false);
-  REQUIRE(sk.get_lg_config_k() == 8);
-  REQUIRE(sk.get_lower_bound(1) == 24.0);
-  REQUIRE(sk.get_estimate() == Approx(24.0).margin(1e-5));
-  REQUIRE(sk.get_upper_bound(1) == Approx(24.001200).margin(1e-5));
-  ifs.close();
-
-  ifs.open(inputPath + "updatable_set_from_java.sk", std::ios::binary);
-  sk = hll_sketch::deserialize(ifs);
-  REQUIRE(sk.is_empty() == false);
-  REQUIRE(sk.get_lg_config_k() == 8);
-  REQUIRE(sk.get_lower_bound(1) == 24.0);
-  REQUIRE(sk.get_estimate() == Approx(24.0).margin(1e-5));
-  REQUIRE(sk.get_upper_bound(1) == Approx(24.001200).margin(1e-5));
-  ifs.close();
-
-
-  ifs.open(inputPath + "array6_from_java.sk", std::ios::binary);
-  sk = hll_sketch::deserialize(ifs);
-  REQUIRE(sk.is_empty() == false);
-  REQUIRE(sk.get_lg_config_k() == 8);
-  REQUIRE(sk.get_lower_bound(1) == Approx(9589.968564).margin(1e-5));
-  REQUIRE(sk.get_estimate() == Approx(10089.150211).margin(1e-5));
-  REQUIRE(sk.get_upper_bound(1) == Approx(10642.370492).margin(1e-5));
-  ifs.close();
-
-
-  ifs.open(inputPath + "compact_array4_from_java.sk", std::ios::binary);
-  sk = hll_sketch::deserialize(ifs);
-  REQUIRE(sk.is_empty() == false);
-  REQUIRE(sk.get_lg_config_k() == 8);
-  REQUIRE(sk.get_lower_bound(1) == Approx(9589.968564).margin(1e-5));
-  REQUIRE(sk.get_estimate() == Approx(10089.150211).margin(1e-5));
-  REQUIRE(sk.get_upper_bound(1) == Approx(10642.370492).margin(1e-5));
-
-  ifs.close();
-
-
-  ifs.open(inputPath + "updatable_array4_from_java.sk", std::ios::binary);
-  sk = hll_sketch::deserialize(ifs);
-  REQUIRE(sk.is_empty() == false);
-  REQUIRE(sk.get_lg_config_k() == 8);
-  REQUIRE(sk.get_lower_bound(1) == Approx(9589.968564).margin(1e-5));
-  REQUIRE(sk.get_estimate() == Approx(10089.150211).margin(1e-5));
-  REQUIRE(sk.get_upper_bound(1) == Approx(10642.370492).margin(1e-5));
-  ifs.close();
-}
-
 static void checkSketchEquality(hll_sketch& sk1, hll_sketch& sk2) {
   REQUIRE(sk1.get_lg_config_k() == sk2.get_lg_config_k());
   REQUIRE(sk1.get_lower_bound(1) == sk2.get_lower_bound(1));