datasketches 0.5.1 → 0.5.3

data/vendor/datasketches-cpp/hll/include/CouponList-internal.hpp

@@ -33,7 +33,7 @@ namespace datasketches {
 
 template<typename A>
 CouponList<A>::CouponList(uint8_t lgConfigK, target_hll_type tgtHllType, hll_mode mode, const A& allocator):
-HllSketchImpl<A>(lgConfigK, tgtHllType, mode, false),
+HllSketchImpl<A>(lgConfigK, tgtHllType, mode),
 couponCount_(0),
 oooFlag_(false),
 coupons_(1ULL << (mode == hll_mode::LIST ? hll_constants::LG_INIT_LIST_SIZE : hll_constants::LG_INIT_SET_SIZE), 0, allocator)
@@ -41,7 +41,7 @@ coupons_(1ULL << (mode == hll_mode::LIST ? hll_constants::LG_INIT_LIST_SIZE : hl
 
 template<typename A>
 CouponList<A>::CouponList(const CouponList& that, const target_hll_type tgtHllType):
-HllSketchImpl<A>(that.lgConfigK_, tgtHllType, that.mode_, false),
+HllSketchImpl<A>(that.lgConfigK_, tgtHllType, that.mode_),
 couponCount_(that.couponCount_),
 oooFlag_(that.oooFlag_),
 coupons_(that.coupons_)
@@ -99,6 +99,13 @@ CouponList<A>* CouponList<A>::newList(const void* bytes, size_t len, const A& al
   const bool emptyFlag = ((data[hll_constants::FLAGS_BYTE] & hll_constants::EMPTY_FLAG_MASK) ? true : false);
 
   const uint32_t couponCount = data[hll_constants::LIST_COUNT_BYTE];
+  // Reject LIST counts at or above the fixed LIST capacity.
+  const uint32_t listCapacity = 1u << hll_constants::LG_INIT_LIST_SIZE;
+  if (couponCount >= listCapacity) {
+    throw std::invalid_argument("Attempt to deserialize invalid CouponList with couponCount >= capacity. Found couponCount: "
+                                + std::to_string(couponCount)
+                                + ", capacity: " + std::to_string(listCapacity));
+  }
   const uint32_t couponsInArray = (compact ? couponCount : (1 << HllUtil<A>::computeLgArrInts(LIST, couponCount, lgK)));
   const size_t expectedLength = hll_constants::LIST_INT_ARR_START + (couponsInArray * sizeof(uint32_t));
   if (len < expectedLength) {
@@ -146,11 +153,19 @@ CouponList<A>* CouponList<A>::newList(std::istream& is, const A& allocator) {
   const bool oooFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::OUT_OF_ORDER_FLAG_MASK) ? true : false);
   const bool emptyFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::EMPTY_FLAG_MASK) ? true : false);
 
+  const uint32_t couponCount = listHeader[hll_constants::LIST_COUNT_BYTE];
+  // Reject LIST counts at or above the fixed LIST capacity.
+  const uint32_t listCapacity = 1u << hll_constants::LG_INIT_LIST_SIZE;
+  if (couponCount >= listCapacity) {
+    throw std::invalid_argument("Attempt to deserialize invalid CouponList with couponCount >= capacity. Found couponCount: "
+                                + std::to_string(couponCount)
+                                + ", capacity: " + std::to_string(listCapacity));
+  }
+
   ClAlloc cla(allocator);
   CouponList<A>* sketch = new (cla.allocate(1)) CouponList<A>(lgK, tgtHllType, mode, allocator);
   using coupon_list_ptr = std::unique_ptr<CouponList<A>, std::function<void(HllSketchImpl<A>*)>>;
   coupon_list_ptr ptr(sketch, sketch->get_deleter());
-  const uint32_t couponCount = listHeader[hll_constants::LIST_COUNT_BYTE];
   sketch->couponCount_ = couponCount;
   sketch->putOutOfOrderFlag(oooFlag); // should always be false for LIST
 
@@ -162,8 +177,7 @@ CouponList<A>* CouponList<A>::newList(std::istream& is, const A& allocator) {
     read(is, sketch->coupons_.data(), numToRead * sizeof(uint32_t));
   }
 
-  if (!is.good())
-    throw std::runtime_error("error reading from std::istream"); 
+  if (!is.good()) { throw std::runtime_error("error reading from std::istream"); } 
 
   return ptr.release();
 }

data/vendor/datasketches-cpp/hll/include/CubicInterpolation-internal.hpp

@@ -165,10 +165,10 @@ static int recursiveFindStraddle(const double xArr[], const int l, const int r,
     throw std::logic_error("target value invariant violated in search");
   }
 
-  if (l+1 == r) return (l);
+  if (l+1 == r) { return (l); }
   m = l + ((r-l)/2);
-  if (xArr[m] <= x) return (recursiveFindStraddle<A>(xArr, m, r, x));
-  else              return (recursiveFindStraddle<A>(xArr, l, m, x));
+  if (xArr[m] <= x) { return (recursiveFindStraddle<A>(xArr, m, r, x)); }
+  else              { return (recursiveFindStraddle<A>(xArr, l, m, x)); }
 }
 
 
@@ -191,7 +191,7 @@ double CubicInterpolation<A>::usingXArrAndYStride(const double xArr[], const int
   const int xArrLenM1 = xArrLen - 1;
 
   if ((xArrLen < 4) || (x < xArr[0]) || (x > xArr[xArrLenM1])) {
-    throw std::logic_error("impossible values during interpolaiton");
+    throw std::logic_error("impossible values during interpolation");
   }
 
   if (x ==  xArr[xArrLenM1]) { /* corner case */

data/vendor/datasketches-cpp/hll/include/HarmonicNumbers-internal.hpp

@@ -21,6 +21,7 @@
 #define _HARMONICNUMBERS_INTERNAL_HPP_
 
 #include "HarmonicNumbers.hpp"
+#include "fdlibm_log.hpp"
 
 #include <cmath>
 
@@ -70,7 +71,7 @@ double HarmonicNumbers<A>::harmonicNumber(const uint64_t x_i) {
   } else {
     double x = static_cast<double>(x_i);
     double invSq = 1.0 / (x * x);
-    double sum = log(x) + EULER_MASCHERONI_CONSTANT + (1.0 / (2.0 * x));
+    double sum = fdlibm::log(x) + EULER_MASCHERONI_CONSTANT + (1.0 / (2.0 * x));
     /* note: the number of terms included from this series expansion is appropriate
        for the size of the exact table (25) and the precision of doubles */
     double pow = invSq; // now n^-2

data/vendor/datasketches-cpp/hll/include/Hll4Array-internal.hpp

@@ -30,8 +30,8 @@
 namespace datasketches {
 
 template<typename A>
-Hll4Array<A>::Hll4Array(uint8_t lgConfigK, bool startFullSize, const A& allocator):
-HllArray<A>(lgConfigK, target_hll_type::HLL_4, startFullSize, allocator),
+Hll4Array<A>::Hll4Array(uint8_t lgConfigK, const A& allocator):
+HllArray<A>(lgConfigK, target_hll_type::HLL_4, allocator),
 auxHashMap_(nullptr)
 {
   const uint32_t numBytes = this->hll4ArrBytes(lgConfigK);
@@ -53,7 +53,7 @@ Hll4Array<A>::Hll4Array(const Hll4Array<A>& that) :
 
 template<typename A>
 Hll4Array<A>::Hll4Array(const HllArray<A>& other) :
-  HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_4, other.isStartFullSize(), other.getAllocator()),
+  HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_4, other.getAllocator()),
   auxHashMap_(nullptr)
 {
   const int numBytes = this->hll4ArrBytes(this->lgConfigK_);
@@ -131,7 +131,7 @@ uint8_t Hll4Array<A>::getSlot(uint32_t slotNo) const {
 
 template<typename A>
 uint8_t Hll4Array<A>::adjustRawValue(uint32_t slot, uint8_t value) const {
-  if (value != hll_constants::AUX_TOKEN) return value + this->curMin_;
+  if (value != hll_constants::AUX_TOKEN) { return value + this->curMin_; }
   return auxHashMap_->mustFindValueFor(slot);
 }
 

data/vendor/datasketches-cpp/hll/include/Hll4Array.hpp

@@ -28,7 +28,7 @@ namespace datasketches {
 template<typename A>
 class Hll4Array final : public HllArray<A> {
   public:
-    explicit Hll4Array(uint8_t lgConfigK, bool startFullSize, const A& allocator);
+    explicit Hll4Array(uint8_t lgConfigK, const A& allocator);
     explicit Hll4Array(const Hll4Array<A>& that);
     explicit Hll4Array(const HllArray<A>& that);
 

data/vendor/datasketches-cpp/hll/include/Hll6Array-internal.hpp

@@ -27,8 +27,8 @@
 namespace datasketches {
 
 template<typename A>
-Hll6Array<A>::Hll6Array(uint8_t lgConfigK, bool startFullSize, const A& allocator):
-HllArray<A>(lgConfigK, target_hll_type::HLL_6, startFullSize, allocator)
+Hll6Array<A>::Hll6Array(uint8_t lgConfigK, const A& allocator):
+HllArray<A>(lgConfigK, target_hll_type::HLL_6, allocator)
 {
   const int numBytes = this->hll6ArrBytes(lgConfigK);
   this->hllByteArr_.resize(numBytes, 0);
@@ -36,7 +36,7 @@ HllArray<A>(lgConfigK, target_hll_type::HLL_6, startFullSize, allocator)
 
 template<typename A>
 Hll6Array<A>::Hll6Array(const HllArray<A>& other) :
-  HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_6, other.isStartFullSize(), other.getAllocator())
+  HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_6, other.getAllocator())
 {
   const int numBytes = this->hll6ArrBytes(this->lgConfigK_);
   this->hllByteArr_.resize(numBytes, 0);

data/vendor/datasketches-cpp/hll/include/Hll6Array.hpp

@@ -30,7 +30,7 @@ class Hll6Iterator;
 template<typename A>
 class Hll6Array final : public HllArray<A> {
   public:
-    Hll6Array(uint8_t lgConfigK, bool startFullSize, const A& allocator);
+    Hll6Array(uint8_t lgConfigK, const A& allocator);
     explicit Hll6Array(const HllArray<A>& that);
 
     virtual ~Hll6Array() = default;

data/vendor/datasketches-cpp/hll/include/Hll8Array-internal.hpp

@@ -25,8 +25,8 @@
 namespace datasketches {
 
 template<typename A>
-Hll8Array<A>::Hll8Array(uint8_t lgConfigK, bool startFullSize, const A& allocator):
-HllArray<A>(lgConfigK, target_hll_type::HLL_8, startFullSize, allocator)
+Hll8Array<A>::Hll8Array(uint8_t lgConfigK, const A& allocator):
+HllArray<A>(lgConfigK, target_hll_type::HLL_8, allocator)
 {
   const int numBytes = this->hll8ArrBytes(lgConfigK);
   this->hllByteArr_.resize(numBytes, 0);
@@ -34,7 +34,7 @@ HllArray<A>(lgConfigK, target_hll_type::HLL_8, startFullSize, allocator)
 
 template<typename A>
 Hll8Array<A>::Hll8Array(const HllArray<A>& other):
-  HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_8, other.isStartFullSize(), other.getAllocator())
+  HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_8, other.getAllocator())
 {
   const int numBytes = this->hll8ArrBytes(this->lgConfigK_);
   this->hllByteArr_.resize(numBytes, 0);
@@ -98,6 +98,9 @@ void Hll8Array<A>::internalCouponUpdate(uint32_t coupon) {
 
   const uint8_t curVal = this->hllByteArr_[slotNo];
   if (newVal > curVal) {
+    // A prior HLL merge may have left the estimator state stale. Rebuild it before applying
+    // an incremental update for a changed register.
+    this->check_rebuild_kxq_cur_min();
     this->hllByteArr_[slotNo] = newVal;
     this->hipAndKxQIncrementalUpdate(curVal, newVal);
     this->numAtCurMin_ -= curVal == 0; // interpret numAtCurMin as num zeros

data/vendor/datasketches-cpp/hll/include/Hll8Array.hpp

@@ -30,7 +30,7 @@ class Hll8Iterator;
 template<typename A>
 class Hll8Array final : public HllArray<A> {
   public:
-    Hll8Array(uint8_t lgConfigK, bool startFullSize, const A& allocator);
+    Hll8Array(uint8_t lgConfigK, const A& allocator);
     explicit Hll8Array(const HllArray<A>& that);
 
     virtual ~Hll8Array() = default;

data/vendor/datasketches-cpp/hll/include/HllArray-internal.hpp

@@ -21,6 +21,7 @@
 #define _HLLARRAY_INTERNAL_HPP_
 
 #include "HllArray.hpp"
+#include "fdlibm_log.hpp"
 #include "HllUtil.hpp"
 #include "HarmonicNumbers.hpp"
 #include "CubicInterpolation.hpp"
@@ -35,8 +36,8 @@
 namespace datasketches {
 
 template<typename A>
-HllArray<A>::HllArray(uint8_t lgConfigK, target_hll_type tgtHllType, bool startFullSize, const A& allocator):
-HllSketchImpl<A>(lgConfigK, tgtHllType, hll_mode::HLL, startFullSize),
+HllArray<A>::HllArray(uint8_t lgConfigK, target_hll_type tgtHllType, const A& allocator):
+HllSketchImpl<A>(lgConfigK, tgtHllType, hll_mode::HLL),
 hipAccum_(0.0),
 kxq0_(1 << lgConfigK),
 kxq1_(0.0),
@@ -49,7 +50,7 @@ rebuild_kxq_curmin_(false)
 
 template<typename A>
 HllArray<A>::HllArray(const HllArray& other, target_hll_type tgtHllType) :
-  HllSketchImpl<A>(other.getLgConfigK(), tgtHllType, hll_mode::HLL, other.isStartFullSize()),
+  HllSketchImpl<A>(other.getLgConfigK(), tgtHllType, hll_mode::HLL),
   // remaining fields are initialized to empty sketch defaults
   // and left to subclass constructor to populate
   hipAccum_(0.0),
@@ -109,7 +110,6 @@ HllArray<A>* HllArray<A>::newHll(const void* bytes, size_t len, const A& allocat
   const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(data[hll_constants::MODE_BYTE]);
   const bool oooFlag = ((data[hll_constants::FLAGS_BYTE] & hll_constants::OUT_OF_ORDER_FLAG_MASK) ? true : false);
   const bool comapctFlag = ((data[hll_constants::FLAGS_BYTE] & hll_constants::COMPACT_FLAG_MASK) ? true : false);
-  const bool startFullSizeFlag = ((data[hll_constants::FLAGS_BYTE] & hll_constants::FULL_SIZE_FLAG_MASK) ? true : false);
 
   const uint8_t lgK = data[hll_constants::LG_K_BYTE];
   const uint8_t curMin = data[hll_constants::HLL_CUR_MIN_BYTE];
@@ -139,18 +139,19 @@ HllArray<A>* HllArray<A>::newHll(const void* bytes, size_t len, const A& allocat
     aux_ptr = aux_hash_map_ptr(auxHashMap, auxHashMap->make_deleter());
   }
 
-  HllArray<A>* sketch = HllSketchImplFactory<A>::newHll(lgK, tgtHllType, startFullSizeFlag, allocator);
+  HllArray<A>* sketch = HllSketchImplFactory<A>::newHll(lgK, tgtHllType, allocator);
   sketch->putCurMin(curMin);
   sketch->putOutOfOrderFlag(oooFlag);
-  if (!oooFlag) sketch->putHipAccum(hip);
+  if (!oooFlag) { sketch->putHipAccum(hip); }
   sketch->putKxQ0(kxq0);
   sketch->putKxQ1(kxq1);
   sketch->putNumAtCurMin(numAtCurMin);
 
   std::memcpy(sketch->hllByteArr_.data(), data + hll_constants::HLL_BYTE_ARR_START, arrayBytes);
 
-  if (auxHashMap != nullptr)
+  if (auxHashMap != nullptr) {
     ((Hll4Array<A>*)sketch)->putAuxHashMap(auxHashMap);
+  }
 
   aux_ptr.release();
   return sketch;
@@ -173,18 +174,17 @@ HllArray<A>* HllArray<A>::newHll(std::istream& is, const A& allocator) {
 
   hll_mode mode = HllSketchImpl<A>::extractCurMode(listHeader[hll_constants::MODE_BYTE]);
   if (mode != HLL) {
-    throw std::invalid_argument("Calling HLL construtor with non-HLL mode data");
+    throw std::invalid_argument("Calling HLL constructor with non-HLL mode data");
   }
 
   const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(listHeader[hll_constants::MODE_BYTE]);
   const bool oooFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::OUT_OF_ORDER_FLAG_MASK) ? true : false);
   const bool comapctFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::COMPACT_FLAG_MASK) ? true : false);
-  const bool startFullSizeFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::FULL_SIZE_FLAG_MASK) ? true : false);
 
   const uint8_t lgK = listHeader[hll_constants::LG_K_BYTE];
   const uint8_t curMin = listHeader[hll_constants::HLL_CUR_MIN_BYTE];
 
-  HllArray* sketch = HllSketchImplFactory<A>::newHll(lgK, tgtHllType, startFullSizeFlag, allocator);
+  HllArray* sketch = HllSketchImplFactory<A>::newHll(lgK, tgtHllType, allocator);
   typedef std::unique_ptr<HllArray<A>, std::function<void(HllSketchImpl<A>*)>> hll_array_ptr;
   hll_array_ptr sketch_ptr(sketch, sketch->get_deleter());
   sketch->putCurMin(curMin);
@@ -193,7 +193,7 @@ HllArray<A>* HllArray<A>::newHll(std::istream& is, const A& allocator) {
   const auto hip = read<double>(is);
   const auto kxq0 = read<double>(is);
   const auto kxq1 = read<double>(is);
-  if (!oooFlag) sketch->putHipAccum(hip);
+  if (!oooFlag) { sketch->putHipAccum(hip); }
   sketch->putKxQ0(kxq0);
   sketch->putKxQ1(kxq1);
 
@@ -209,8 +209,7 @@ HllArray<A>* HllArray<A>::newHll(std::istream& is, const A& allocator) {
     ((Hll4Array<A>*)sketch)->putAuxHashMap(auxHashMap);
   }
 
-  if (!is.good())
-    throw std::runtime_error("error reading from std::istream"); 
+  if (!is.good()) { throw std::runtime_error("error reading from std::istream"); } 
 
   return sketch_ptr.release();
 }
@@ -465,6 +464,10 @@ bool HllArray<A>::isCompact() const {
 
 template<typename A>
 bool HllArray<A>::isEmpty() const {
+  // mergeHll() is only called with non-empty sketches and sets this flag after updating the
+  // register array. The cached curMin_/numAtCurMin_ may still have their empty-sketch values,
+  // but a pending rebuild therefore proves that this array is not empty.
+  if (rebuild_kxq_curmin_) return false;
   const uint32_t configK = 1 << this->lgConfigK_;
   return (curMin_ == 0) && (numAtCurMin_ == configK);
 }
@@ -545,7 +548,7 @@ template<typename A>
 void HllArray<A>::hipAndKxQIncrementalUpdate(uint8_t oldValue, uint8_t newValue) {
   const uint32_t configK = 1 << this->getLgConfigK();
   // update hip BEFORE updating kxq
-  if (!oooFlag_) hipAccum_ += configK / (kxq0_ + kxq1_);
+  if (!oooFlag_) { hipAccum_ += configK / (kxq0_ + kxq1_); }
   // update kxq0 and kxq1; subtract first, then add
   if (oldValue < 32) { kxq0_ -= INVERSE_POWERS_OF_2[oldValue]; }
   else               { kxq1_ -= INVERSE_POWERS_OF_2[oldValue]; }
@@ -566,7 +569,7 @@ double HllArray<A>::getHllBitMapEstimate() const {
 
   //This will eventually go away.
   if (numUnhitBuckets == 0) {
-    return configK * log(configK / 0.5);
+    return configK * fdlibm::log(configK / 0.5);
   }
 
   const uint32_t numHitBuckets = configK - numUnhitBuckets;
@@ -599,34 +602,37 @@ bool HllArray<A>::isRebuildKxqCurminFlag() const {
 template<typename A>
 void HllArray<A>::check_rebuild_kxq_cur_min() {
   if (!rebuild_kxq_curmin_) { return; }
+  // the deferred rebuild is only ever set on an HLL_8 union gadget. Guarding here also keeps
+  // this from ever rewriting curMin_ on an HLL_4 array, whose nibbles are stored relative to it.
+  if (this->getCurMode() != hll_mode::HLL || this->getTgtHllType() != target_hll_type::HLL_8) {
+    rebuild_kxq_curmin_ = false;
+    return;
+  }
 
-  uint8_t cur_min = 64;
-  uint32_t num_at_cur_min = 0;
+  uint32_t num_zeros = 0;
   double kxq0 = 1 << this->lgConfigK_;
   double kxq1 = 0;
 
-  auto it = this->begin(true); // want all points to adjust cur_min
+  auto it = this->begin(true); // want all slots, including the empty ones
   const auto end = this->end();
   while (it != end) {
     uint8_t v = HllUtil<A>::getValue(*it);
     if (v > 0) {
       if (v < 32) { kxq0 += INVERSE_POWERS_OF_2[v] - 1.0; }
       else        { kxq1 += INVERSE_POWERS_OF_2[v] - 1.0; }
-    }
-    if (v > cur_min) { ++it; continue; }
-    if (v < cur_min) {
-      cur_min = v;
-      num_at_cur_min = 1;
     } else {
-      ++num_at_cur_min;
-    }    
+      ++num_zeros;
+    }
     ++it;
   }
 
   kxq0_ = kxq0;
   kxq1_ = kxq1;
-  curMin_ = cur_min;
-  numAtCurMin_ = num_at_cur_min;
+  // HLL_8 convention: curMin is always 0 and numAtCurMin is the number of zero registers.
+  // That is the representation the incremental update path maintains, so the rebuilt state is
+  // indistinguishable from it and the timing of this rebuild is not observable in the image.
+  curMin_ = 0;
+  numAtCurMin_ = num_zeros;
   rebuild_kxq_curmin_ = false;
   // HipAccum is not affected
 
@@ -648,7 +654,7 @@ array_(array), array_size_(array_size), index_(index), hll_type_(hll_type), exce
 {
   while (index_ < array_size_) {
     value_ = get_value(array_, index_, hll_type_, exceptions_, offset_);
-    if (all_ || value_ != hll_constants::EMPTY) break;
+    if (all_ || value_ != hll_constants::EMPTY) { break; }
     ++index_;
   }
 }
@@ -657,7 +663,7 @@ template<typename A>
 typename HllArray<A>::const_iterator& HllArray<A>::const_iterator::operator++() {
   while (++index_ < array_size_) {
     value_ = get_value(array_, index_, hll_type_, exceptions_, offset_);
-    if (all_ || value_ != hll_constants::EMPTY) break;
+    if (all_ || value_ != hll_constants::EMPTY) { break; }
   }
   return *this;
 }

data/vendor/datasketches-cpp/hll/include/HllArray.hpp

@@ -33,7 +33,7 @@ class HllArray : public HllSketchImpl<A> {
   public:
     using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
 
-    HllArray(uint8_t lgConfigK, target_hll_type tgtHllType, bool startFullSize, const A& allocator);
+    HllArray(uint8_t lgConfigK, target_hll_type tgtHllType, const A& allocator);
     explicit HllArray(const HllArray& other, target_hll_type tgtHllType);
 
     static HllArray* newHll(const void* bytes, size_t len, const A& allocator);

data/vendor/datasketches-cpp/hll/include/HllSketch-internal.hpp

@@ -45,7 +45,7 @@ template<typename A>
 hll_sketch_alloc<A>::hll_sketch_alloc(uint8_t lg_config_k, target_hll_type tgt_type, bool start_full_size, const A& allocator) {
   HllUtil<A>::checkLgK(lg_config_k);
   if (start_full_size) {
-    sketch_impl = HllSketchImplFactory<A>::newHll(lg_config_k, tgt_type, start_full_size, allocator);
+    sketch_impl = HllSketchImplFactory<A>::newHll(lg_config_k, tgt_type, allocator);
   } else {
     typedef typename std::allocator_traits<A>::template rebind_alloc<CouponList<A>> clAlloc;
     sketch_impl = new (clAlloc(allocator).allocate(1)) CouponList<A>(lg_config_k, tgt_type, hll_mode::LIST, allocator);
@@ -107,10 +107,8 @@ hll_sketch_alloc<A>& hll_sketch_alloc<A>::operator=(hll_sketch_alloc<A>&& other)
 }
 
 template<typename A>
-void hll_sketch_alloc<A>::reset() {
-  // TODO: need to allow starting from a full-sized sketch
-  //       (either here or in other implementation)
-  sketch_impl = sketch_impl->reset();
+void hll_sketch_alloc<A>::reset(bool full_size) {
+  sketch_impl = sketch_impl->reset(full_size);
 }
 
 template<typename A>

data/vendor/datasketches-cpp/hll/include/HllSketchImpl-internal.hpp

@@ -29,11 +29,10 @@ namespace datasketches {
 
 template<typename A>
 HllSketchImpl<A>::HllSketchImpl(uint8_t lgConfigK, target_hll_type tgtHllType,
-                                hll_mode mode, bool startFullSize)
+                                hll_mode mode)
   : lgConfigK_(lgConfigK),
     tgtHllType_(tgtHllType),
-    mode_(mode),
-    startFullSize_(startFullSize)
+    mode_(mode)
 {
 }
 
@@ -75,7 +74,7 @@ uint8_t HllSketchImpl<A>::makeFlagsByte(bool compact) const {
   flags |= (isEmpty() ? hll_constants::EMPTY_FLAG_MASK : 0);
   flags |= (compact ? hll_constants::COMPACT_FLAG_MASK : 0);
   flags |= (isOutOfOrderFlag() ? hll_constants::OUT_OF_ORDER_FLAG_MASK : 0);
-  flags |= (startFullSize_ ? hll_constants::FULL_SIZE_FLAG_MASK : 0);
+  // bit 32 is reserved: see RESERVED_FLAG_MASK_32 in HllUtil.hpp
   return flags;
 }
 
@@ -122,8 +121,8 @@ uint8_t HllSketchImpl<A>::makeModeByte() const {
 }
 
 template<typename A>
-HllSketchImpl<A>* HllSketchImpl<A>::reset() {
-  return HllSketchImplFactory<A>::reset(this, startFullSize_);
+HllSketchImpl<A>* HllSketchImpl<A>::reset(bool full_size) {
+  return HllSketchImplFactory<A>::reset(this, full_size);
 }
 
 template<typename A>
@@ -141,11 +140,6 @@ hll_mode HllSketchImpl<A>::getCurMode() const {
   return mode_;
 }
 
-template<typename A>
-bool HllSketchImpl<A>::isStartFullSize() const {
-  return startFullSize_;
-}
-
 }
 
 #endif // _HLLSKETCHIMPL_INTERNAL_HPP_

data/vendor/datasketches-cpp/hll/include/HllSketchImpl.hpp

@@ -32,7 +32,7 @@ class HllSketchImpl {
   public:
     using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
 
-    HllSketchImpl(uint8_t lgConfigK, target_hll_type tgtHllType, hll_mode mode, bool startFullSize);
+    HllSketchImpl(uint8_t lgConfigK, target_hll_type tgtHllType, hll_mode mode);
     virtual ~HllSketchImpl();
 
     virtual void serialize(std::ostream& os, bool compact) const = 0;
@@ -40,7 +40,7 @@ class HllSketchImpl {
 
     virtual HllSketchImpl* copy() const = 0;
     virtual HllSketchImpl* copyAs(target_hll_type tgtHllType) const = 0;
-    HllSketchImpl<A>* reset();
+    HllSketchImpl<A>* reset(bool full_size);
 
     virtual std::function<void(HllSketchImpl<A>*)> get_deleter() const = 0;
 
@@ -69,7 +69,6 @@ class HllSketchImpl {
     virtual bool isOutOfOrderFlag() const = 0;
     virtual void putOutOfOrderFlag(bool oooFlag) = 0;
     virtual A getAllocator() const = 0;
-    bool isStartFullSize() const;
 
   protected:
     static target_hll_type extractTgtHllType(uint8_t modeByte);
@@ -80,7 +79,6 @@ class HllSketchImpl {
     const uint8_t lgConfigK_;
     const target_hll_type tgtHllType_;
     const hll_mode mode_;
-    const bool startFullSize_;
 };
 
 }

data/vendor/datasketches-cpp/hll/include/HllSketchImplFactory.hpp

@@ -41,10 +41,11 @@ public:
 
   static CouponHashSet<A>* promoteListToSet(const CouponList<A>& list);
   static HllArray<A>* promoteListOrSetToHll(const CouponList<A>& list);
-  static HllArray<A>* newHll(uint8_t lgConfigK, target_hll_type tgtHllType, bool startFullSize, const A& allocator);
+  static HllArray<A>* newHll(uint8_t lgConfigK, target_hll_type tgtHllType, const A& allocator);
   
   // resets the input impl, deleting the input pointer and returning a new pointer
-  static HllSketchImpl<A>* reset(HllSketchImpl<A>* impl, bool startFullSize);
+  // full_size selects the state to reset to: an empty HLL array, or LIST (coupon) mode
+  static HllSketchImpl<A>* reset(HllSketchImpl<A>* impl, bool full_size);
 
   static Hll4Array<A>* convertToHll4(const HllArray<A>& srcHllArr);
   static Hll6Array<A>* convertToHll6(const HllArray<A>& srcHllArr);
@@ -63,7 +64,7 @@ CouponHashSet<A>* HllSketchImplFactory<A>::promoteListToSet(const CouponList<A>&
 
 template<typename A>
 HllArray<A>* HllSketchImplFactory<A>::promoteListOrSetToHll(const CouponList<A>& src) {
-  HllArray<A>* tgtHllArr = HllSketchImplFactory<A>::newHll(src.getLgConfigK(), src.getTgtHllType(), false, src.getAllocator());
+  HllArray<A>* tgtHllArr = HllSketchImplFactory<A>::newHll(src.getLgConfigK(), src.getTgtHllType(), src.getAllocator());
   tgtHllArr->putKxQ0(1 << src.getLgConfigK());
   for (const auto coupon: src) {
     tgtHllArr->couponUpdate(coupon);
@@ -105,25 +106,25 @@ HllSketchImpl<A>* HllSketchImplFactory<A>::deserialize(const void* bytes, size_t
 }
 
 template<typename A>
-HllArray<A>* HllSketchImplFactory<A>::newHll(uint8_t lgConfigK, target_hll_type tgtHllType, bool startFullSize, const A& allocator) {
+HllArray<A>* HllSketchImplFactory<A>::newHll(uint8_t lgConfigK, target_hll_type tgtHllType, const A& allocator) {
   switch (tgtHllType) {
     case HLL_8:
       using Hll8Alloc = typename std::allocator_traits<A>::template rebind_alloc<Hll8Array<A>>;
-      return new (Hll8Alloc(allocator).allocate(1)) Hll8Array<A>(lgConfigK, startFullSize, allocator);
+      return new (Hll8Alloc(allocator).allocate(1)) Hll8Array<A>(lgConfigK, allocator);
     case HLL_6:
       using Hll6Alloc = typename std::allocator_traits<A>::template rebind_alloc<Hll6Array<A>>;
-      return new (Hll6Alloc(allocator).allocate(1)) Hll6Array<A>(lgConfigK, startFullSize, allocator);
+      return new (Hll6Alloc(allocator).allocate(1)) Hll6Array<A>(lgConfigK, allocator);
     case HLL_4:
       using Hll4Alloc = typename std::allocator_traits<A>::template rebind_alloc<Hll4Array<A>>;
-      return new (Hll4Alloc(allocator).allocate(1)) Hll4Array<A>(lgConfigK, startFullSize, allocator);
+      return new (Hll4Alloc(allocator).allocate(1)) Hll4Array<A>(lgConfigK, allocator);
   }
   throw std::logic_error("Invalid target_hll_type");
 }
 
 template<typename A>
-HllSketchImpl<A>* HllSketchImplFactory<A>::reset(HllSketchImpl<A>* impl, bool startFullSize) {
-  if (startFullSize) {
-    HllArray<A>* hll = newHll(impl->getLgConfigK(), impl->getTgtHllType(), startFullSize, impl->getAllocator());
+HllSketchImpl<A>* HllSketchImplFactory<A>::reset(HllSketchImpl<A>* impl, bool full_size) {
+  if (full_size) {
+    HllArray<A>* hll = newHll(impl->getLgConfigK(), impl->getTgtHllType(), impl->getAllocator());
     impl->get_deleter()(impl);
     return hll;
   } else {

data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp

@@ -39,18 +39,21 @@ hll_union_alloc<A>::hll_union_alloc(uint8_t lg_max_k, const A& allocator):
 
 template<typename A>
 hll_sketch_alloc<A> hll_union_alloc<A>::get_result(target_hll_type target_type) const {
+  if (gadget_.sketch_impl->getCurMode() == hll_mode::HLL) {
+    static_cast<HllArray<A>*>(gadget_.sketch_impl)->check_rebuild_kxq_cur_min();
+  }
   return hll_sketch_alloc<A>(gadget_, target_type);
 }
 
 template<typename A>
 void hll_union_alloc<A>::update(const hll_sketch_alloc<A>& sketch) {
-  if (sketch.is_empty()) return;
+  if (sketch.is_empty()) { return; }
   union_impl(sketch, lg_max_k_);
 }
 
 template<typename A>
 void hll_union_alloc<A>::update(hll_sketch_alloc<A>&& sketch) {
-  if (sketch.is_empty()) return;
+  if (sketch.is_empty()) { return; }
   if (gadget_.is_empty() && sketch.get_target_type() == HLL_8 && sketch.get_lg_config_k() <= lg_max_k_) {
     if (sketch.get_current_mode() == HLL || sketch.get_lg_config_k() == lg_max_k_) {
       gadget_ = std::move(sketch);
@@ -131,29 +134,33 @@ void hll_union_alloc<A>::coupon_update(uint32_t coupon) {
 
 template<typename A>
 double hll_union_alloc<A>::get_estimate() const {
-  if (gadget_.sketch_impl->getCurMode() == hll_mode::HLL)
+  if (gadget_.sketch_impl->getCurMode() == hll_mode::HLL) {
     static_cast<HllArray<A>*>(gadget_.sketch_impl)->check_rebuild_kxq_cur_min();
+  }
   return gadget_.get_estimate();
 }
 
 template<typename A>
 double hll_union_alloc<A>::get_composite_estimate() const {
-  if (gadget_.sketch_impl->getCurMode() == hll_mode::HLL)
+  if (gadget_.sketch_impl->getCurMode() == hll_mode::HLL) {
     static_cast<HllArray<A>*>(gadget_.sketch_impl)->check_rebuild_kxq_cur_min();
+  }
   return gadget_.get_composite_estimate();
 }
 
 template<typename A>
 double hll_union_alloc<A>::get_lower_bound(uint8_t num_std_dev) const {
-  if (gadget_.sketch_impl->getCurMode() == hll_mode::HLL)
+  if (gadget_.sketch_impl->getCurMode() == hll_mode::HLL) {
     static_cast<HllArray<A>*>(gadget_.sketch_impl)->check_rebuild_kxq_cur_min();
+  }
   return gadget_.get_lower_bound(num_std_dev);
 }
 
 template<typename A>
 double hll_union_alloc<A>::get_upper_bound(uint8_t num_std_dev) const {
-  if (gadget_.sketch_impl->getCurMode() == hll_mode::HLL)
+  if (gadget_.sketch_impl->getCurMode() == hll_mode::HLL) {
     static_cast<HllArray<A>*>(gadget_.sketch_impl)->check_rebuild_kxq_cur_min();
+  }
   return gadget_.get_upper_bound(num_std_dev);
 }
 
@@ -164,7 +171,9 @@ uint8_t hll_union_alloc<A>::get_lg_config_k() const {
 
 template<typename A>
 void hll_union_alloc<A>::reset() {
-  gadget_.reset();
+  // always coupon collection mode: the gadget is an internal detail and must not
+  // inherit a full-size state from whatever sketches happen to have been unioned in
+  gadget_.reset(false);
 }
 
 template<typename A>
@@ -209,7 +218,7 @@ HllSketchImpl<A>* hll_union_alloc<A>::copy_or_downsample(const HllSketchImpl<A>*
     return src->copyAs(HLL_8);
   }
   typedef typename std::allocator_traits<A>::template rebind_alloc<Hll8Array<A>> hll8Alloc;
-  Hll8Array<A>* tgtHllArr = new (hll8Alloc(src->getAllocator()).allocate(1)) Hll8Array<A>(tgt_lg_k, false, src->getAllocator());
+  Hll8Array<A>* tgtHllArr = new (hll8Alloc(src->getAllocator()).allocate(1)) Hll8Array<A>(tgt_lg_k, src->getAllocator());
   tgtHllArr->mergeHll(*src);
   //both of these are required for isomorphism
   tgtHllArr->putHipAccum(src->getHipAccum());