datasketches 0.3.1 → 0.4.0

data/vendor/datasketches-cpp/hll/include/CouponList.hpp

@@ -33,12 +33,14 @@ class HllSketchImplFactory;
 template<typename A>
 class CouponList : public HllSketchImpl<A> {
   public:
+    using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
+
     CouponList(uint8_t lgConfigK, target_hll_type tgtHllType, hll_mode mode, const A& allocator);
     CouponList(const CouponList& that, target_hll_type tgtHllType);
 
     static CouponList* newList(const void* bytes, size_t len, const A& allocator);
     static CouponList* newList(std::istream& is, const A& allocator);
-    virtual vector_u8<A> serialize(bool compact, unsigned header_size_bytes) const;
+    virtual vector_bytes serialize(bool compact, unsigned header_size_bytes) const;
     virtual void serialize(std::ostream& os, bool compact) const;
 
     virtual ~CouponList() = default;

data/vendor/datasketches-cpp/hll/include/Hll4Array-internal.hpp

@@ -51,6 +51,22 @@ Hll4Array<A>::Hll4Array(const Hll4Array<A>& that) :
   }
 }
 
+template<typename A>
+Hll4Array<A>::Hll4Array(const HllArray<A>& other) :
+  HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_4, other.isStartFullSize(), other.getAllocator()),
+  auxHashMap_(nullptr)
+{
+  const int numBytes = this->hll4ArrBytes(this->lgConfigK_);
+  this->hllByteArr_.resize(numBytes, 0);
+  this->oooFlag_ = other.isOutOfOrderFlag();
+
+  for (const auto coupon : other) { // all = false, so skip empty values
+    internalCouponUpdate(coupon); // updates KxQ registers
+  }
+  this->hipAccum_ = other.getHipAccum();
+  this->rebuild_kxq_curmin_ = false;
+}
+
 template<typename A>
 Hll4Array<A>::~Hll4Array() {
   // hllByteArr deleted in parent
@@ -114,10 +130,9 @@ uint8_t Hll4Array<A>::getSlot(uint32_t slotNo) const {
 }
 
 template<typename A>
-uint8_t Hll4Array<A>::get_value(uint32_t index) const {
-  const uint8_t value = getSlot(index);
+uint8_t Hll4Array<A>::adjustRawValue(uint32_t slot, uint8_t value) const {
   if (value != hll_constants::AUX_TOKEN) return value + this->curMin_;
-  return auxHashMap_->mustFindValueFor(index);
+  return auxHashMap_->mustFindValueFor(slot);
 }
 
 template<typename A>
@@ -210,7 +225,7 @@ void Hll4Array<A>::internalHll4Update(uint32_t slotNo, uint8_t newVal) {
 
       // we just increased a pair value, so it might be time to change curMin
       if (actualOldValue == this->curMin_) { // 908
-        this->decNumAtCurMin();
+        --(this->numAtCurMin_);
         while (this->numAtCurMin_ == 0) {
           shiftToBiggerCurMin(); // increases curMin by 1, builds a new aux table
           // shifts values in 4-bit table and recounts curMin
@@ -328,13 +343,6 @@ typename HllArray<A>::const_iterator Hll4Array<A>::end() const {
       this->tgtHllType_, auxHashMap_, this->curMin_, false);
 }
 
-template<typename A>
-void Hll4Array<A>::mergeHll(const HllArray<A>& src) {
-  for (const auto coupon: src) {
-    internalCouponUpdate(coupon);
-  }
-}
-
 }
 
 #endif // _HLL4ARRAY_INTERNAL_HPP_

data/vendor/datasketches-cpp/hll/include/Hll4Array.hpp

@@ -25,14 +25,12 @@
 
 namespace datasketches {
 
-template<typename A>
-class Hll4Iterator;
-
 template<typename A>
 class Hll4Array final : public HllArray<A> {
   public:
     explicit Hll4Array(uint8_t lgConfigK, bool startFullSize, const A& allocator);
     explicit Hll4Array(const Hll4Array<A>& that);
+    explicit Hll4Array(const HllArray<A>& that);
 
     virtual ~Hll4Array();
     virtual std::function<void(HllSketchImpl<A>*)> get_deleter() const;
@@ -41,13 +39,12 @@ class Hll4Array final : public HllArray<A> {
 
     inline uint8_t getSlot(uint32_t slotNo) const;
     inline void putSlot(uint32_t slotNo, uint8_t value);
-    inline uint8_t get_value(uint32_t index) const;
+    inline uint8_t adjustRawValue(uint32_t index, uint8_t value) const;
 
     virtual uint32_t getUpdatableSerializationBytes() const;
     virtual uint32_t getHllByteArrBytes() const;
 
     virtual HllSketchImpl<A>* couponUpdate(uint32_t coupon) final;
-    void mergeHll(const HllArray<A>& src);
 
     virtual AuxHashMap<A>* getAuxHashMap() const;
     // does *not* delete old map if overwriting

data/vendor/datasketches-cpp/hll/include/Hll6Array-internal.hpp

@@ -34,6 +34,25 @@ HllArray<A>(lgConfigK, target_hll_type::HLL_6, startFullSize, allocator)
   this->hllByteArr_.resize(numBytes, 0);
 }
 
+template<typename A>
+Hll6Array<A>::Hll6Array(const HllArray<A>& other) :
+  HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_6, other.isStartFullSize(), other.getAllocator())
+{
+  const int numBytes = this->hll6ArrBytes(this->lgConfigK_);
+  this->hllByteArr_.resize(numBytes, 0);
+  this->oooFlag_ = other.isOutOfOrderFlag();
+  uint32_t num_zeros = 1 << this->lgConfigK_;
+  
+  for (const auto coupon : other) { // all = false, so skip empty values
+    num_zeros--;
+    internalCouponUpdate(coupon); // updates KxQ registers
+  }
+  
+  this->numAtCurMin_ = num_zeros;
+  this->hipAccum_ = other.getHipAccum();
+  this->rebuild_kxq_curmin_ = false;
+}
+
 template<typename A>
 std::function<void(HllSketchImpl<A>*)> Hll6Array<A>::get_deleter() const {
   return [](HllSketchImpl<A>* ptr) {
@@ -101,13 +120,6 @@ void Hll6Array<A>::internalCouponUpdate(uint32_t coupon) {
   }
 }
 
-template<typename A>
-void Hll6Array<A>::mergeHll(const HllArray<A>& src) {
-  for (const auto coupon: src) {
-    internalCouponUpdate(coupon);
-  }
-}
-
 }
 
 #endif // _HLL6ARRAY_INTERNAL_HPP_

data/vendor/datasketches-cpp/hll/include/Hll6Array.hpp

@@ -31,6 +31,7 @@ template<typename A>
 class Hll6Array final : public HllArray<A> {
   public:
     Hll6Array(uint8_t lgConfigK, bool startFullSize, const A& allocator);
+    explicit Hll6Array(const HllArray<A>& that);
 
     virtual ~Hll6Array() = default;
     virtual std::function<void(HllSketchImpl<A>*)> get_deleter() const;
@@ -41,7 +42,6 @@ class Hll6Array final : public HllArray<A> {
     inline void putSlot(uint32_t slotNo, uint8_t value);
 
     virtual HllSketchImpl<A>* couponUpdate(uint32_t coupon) final;
-    void mergeHll(const HllArray<A>& src);
 
     virtual uint32_t getHllByteArrBytes() const;
 

data/vendor/datasketches-cpp/hll/include/Hll8Array-internal.hpp

@@ -32,6 +32,25 @@ HllArray<A>(lgConfigK, target_hll_type::HLL_8, startFullSize, allocator)
   this->hllByteArr_.resize(numBytes, 0);
 }
 
+template<typename A>
+Hll8Array<A>::Hll8Array(const HllArray<A>& other):
+  HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_8, other.isStartFullSize(), other.getAllocator())
+{
+  const int numBytes = this->hll8ArrBytes(this->lgConfigK_);
+  this->hllByteArr_.resize(numBytes, 0);
+  this->oooFlag_ = other.isOutOfOrderFlag();
+  uint32_t num_zeros = 1 << this->lgConfigK_;
+  
+  for (const auto coupon : other) { // all = false, so skip empty values
+    num_zeros--;
+    internalCouponUpdate(coupon); // updates KxQ registers
+  }
+  
+  this->numAtCurMin_ = num_zeros;
+  this->hipAccum_ = other.getHipAccum();
+  this->rebuild_kxq_curmin_ = false;
+}
+
 template<typename A>
 std::function<void(HllSketchImpl<A>*)> Hll8Array<A>::get_deleter() const {
   return [](HllSketchImpl<A>* ptr) {
@@ -77,13 +96,11 @@ void Hll8Array<A>::internalCouponUpdate(uint32_t coupon) {
   const uint32_t slotNo = HllUtil<A>::getLow26(coupon) & configKmask;
   const uint8_t newVal = HllUtil<A>::getValue(coupon);
 
-  const uint8_t curVal = getSlot(slotNo);
+  const uint8_t curVal = this->hllByteArr_[slotNo];
   if (newVal > curVal) {
-    putSlot(slotNo, newVal);
+    this->hllByteArr_[slotNo] = newVal;
     this->hipAndKxQIncrementalUpdate(curVal, newVal);
-    if (curVal == 0) {
-      this->numAtCurMin_--; // interpret numAtCurMin as num zeros
-    }
+    this->numAtCurMin_ -= curVal == 0; // interpret numAtCurMin as num zeros
   }
 }
 
@@ -97,49 +114,88 @@ void Hll8Array<A>::mergeList(const CouponList<A>& src) {
 template<typename A>
 void Hll8Array<A>::mergeHll(const HllArray<A>& src) {
   // at this point src_k >= dst_k
-  const uint32_t src_k = 1 << src.getLgConfigK();
-  const uint32_t dst_mask = (1 << this->getLgConfigK()) - 1;
-  // duplication below is to avoid a virtual method call in a loop
-  if (src.getTgtHllType() == target_hll_type::HLL_8) {
-    for (uint32_t i = 0; i < src_k; i++) {
-      const uint8_t new_v = static_cast<const Hll8Array<A>&>(src).getSlot(i);
-      const uint32_t j = i & dst_mask;
-      const uint8_t old_v = this->hllByteArr_[j];
-      if (new_v > old_v) {
-        this->hllByteArr_[j] = new_v;
-        this->hipAndKxQIncrementalUpdate(old_v, new_v);
-        if (old_v == 0) {
-          this->numAtCurMin_--;
-        }
+  // we can optimize further when the k values are equal
+  if (this->getLgConfigK() == src.getLgConfigK()) {
+    if (src.getTgtHllType() == target_hll_type::HLL_8) {
+      uint32_t i = 0;
+      for (const auto value: src.getHllArray()) {
+        this->hllByteArr_[i] = std::max(this->hllByteArr_[i], value);
+        ++i;
       }
-    }
-  } else if (src.getTgtHllType() == target_hll_type::HLL_6) {
-    for (uint32_t i = 0; i < src_k; i++) {
-      const uint8_t new_v = static_cast<const Hll6Array<A>&>(src).getSlot(i);
-      const uint32_t j = i & dst_mask;
-      const uint8_t old_v = this->hllByteArr_[j];
-      if (new_v > old_v) {
-        this->hllByteArr_[j] = new_v;
-        this->hipAndKxQIncrementalUpdate(old_v, new_v);
-        if (old_v == 0) {
-          this->numAtCurMin_--;
-        }
+    } else if (src.getTgtHllType() == target_hll_type::HLL_6) {
+      const uint32_t src_k = 1 << src.getLgConfigK();
+      uint32_t i = 0;
+      const uint8_t* ptr = src.getHllArray().data();
+      while (i < src_k) {
+        uint8_t value = *ptr & 0x3f;
+        this->hllByteArr_[i] = std::max(this->hllByteArr_[i], value);
+        ++i;
+        value = *ptr++ >> 6;
+        value |= (*ptr & 0x0f) << 2;
+        this->hllByteArr_[i] = std::max(this->hllByteArr_[i], value);
+        ++i;
+        value = *ptr++ >> 4;
+        value |= (*ptr & 3) << 4;
+        this->hllByteArr_[i] = std::max(this->hllByteArr_[i], value);
+        ++i;
+        value = *ptr++ >> 2;
+        this->hllByteArr_[i] = std::max(this->hllByteArr_[i], value);
+        ++i;
+      }
+    } else { // HLL_4
+      const auto& src4 = static_cast<const Hll4Array<A>&>(src);
+      uint32_t i = 0;
+      for (const auto byte: src.getHllArray()) {
+        this->hllByteArr_[i] = std::max(this->hllByteArr_[i], src4.adjustRawValue(i, byte & hll_constants::loNibbleMask));
+        ++i;
+        this->hllByteArr_[i] = std::max(this->hllByteArr_[i], src4.adjustRawValue(i, byte >> 4));
+        ++i;
       }
     }
-  } else { // HLL_4
-    for (uint32_t i = 0; i < src_k; i++) {
-      const uint8_t new_v = static_cast<const Hll4Array<A>&>(src).get_value(i);
-      const uint32_t j = i & dst_mask;
-      const uint8_t old_v = this->hllByteArr_[j];
-      if (new_v > old_v) {
-        this->hllByteArr_[j] = new_v;
-        this->hipAndKxQIncrementalUpdate(old_v, new_v);
-        if (old_v == 0) {
-          this->numAtCurMin_--;
-        }
+  } else {
+    // src_k > dst_k
+    const uint32_t dst_mask = (1 << this->getLgConfigK()) - 1;
+    // special treatment below to optimize performance
+    if (src.getTgtHllType() == target_hll_type::HLL_8) {
+      uint32_t i = 0;
+      for (const auto value: src.getHllArray()) {
+        processValue(i++, dst_mask, value);
+      }
+    } else if (src.getTgtHllType() == target_hll_type::HLL_6) {
+      const uint32_t src_k = 1 << src.getLgConfigK();
+      uint32_t i = 0;
+      const uint8_t* ptr = src.getHllArray().data();
+      while (i < src_k) {
+        uint8_t value = *ptr & 0x3f;
+        processValue(i++, dst_mask, value);
+        value = *ptr++ >> 6;
+        value |= (*ptr & 0x0f) << 2;
+        processValue(i++, dst_mask, value);
+        value = *ptr++ >> 4;
+        value |= (*ptr & 3) << 4;
+        processValue(i++, dst_mask, value);
+        value = *ptr++ >> 2;
+        processValue(i++, dst_mask, value);
+      }
+    } else { // HLL_4
+      const auto& src4 = static_cast<const Hll4Array<A>&>(src);
+      uint32_t i = 0;
+      for (const auto byte: src.getHllArray()) {
+        processValue(i, dst_mask, src4.adjustRawValue(i, byte & hll_constants::loNibbleMask));
+        ++i;
+        processValue(i, dst_mask, src4.adjustRawValue(i, byte >> 4));
+        ++i;
       }
     }
   }
+  this->setRebuildKxqCurminFlag(true);
+}
+
+
+template<typename A>
+void Hll8Array<A>::processValue(uint32_t slot, uint32_t mask, uint8_t new_val) {
+  const size_t index = slot & mask;
+  this->hllByteArr_[index] = std::max(this->hllByteArr_[index], new_val);
 }
 
 }

data/vendor/datasketches-cpp/hll/include/Hll8Array.hpp

@@ -31,6 +31,7 @@ template<typename A>
 class Hll8Array final : public HllArray<A> {
   public:
     Hll8Array(uint8_t lgConfigK, bool startFullSize, const A& allocator);
+    explicit Hll8Array(const HllArray<A>& that);
 
     virtual ~Hll8Array() = default;
     virtual std::function<void(HllSketchImpl<A>*)> get_deleter() const;
@@ -48,6 +49,7 @@ class Hll8Array final : public HllArray<A> {
 
   private:
     inline void internalCouponUpdate(uint32_t coupon);
+    inline void processValue(uint32_t slot, uint32_t mask, uint8_t new_val);
 };
 
 }

data/vendor/datasketches-cpp/hll/include/HllArray-internal.hpp

@@ -43,20 +43,44 @@ kxq1_(0.0),
 hllByteArr_(allocator),
 curMin_(0),
 numAtCurMin_(1 << lgConfigK),
-oooFlag_(false)
+oooFlag_(false),
+rebuild_kxq_curmin_(false)
+{}
+
+template<typename A>
+HllArray<A>::HllArray(const HllArray& other, target_hll_type tgtHllType) :
+  HllSketchImpl<A>(other.getLgConfigK(), tgtHllType, hll_mode::HLL, other.isStartFullSize()),
+  // remaining fields are initialized to empty sketch defaults
+  // and left to subclass constructor to populate
+  hipAccum_(0.0),
+  kxq0_(1 << other.getLgConfigK()),
+  kxq1_(0.0),
+  hllByteArr_(other.getAllocator()),
+  curMin_(0),
+  numAtCurMin_(1 << other.getLgConfigK()),
+  oooFlag_(false),
+  rebuild_kxq_curmin_(false)
 {}
 
 template<typename A>
 HllArray<A>* HllArray<A>::copyAs(target_hll_type tgtHllType) const {
-  if (tgtHllType == this->getTgtHllType()) {
+  // we may need to recompute KxQ and curMin data for a union gadget,
+  // so only use a direct copy if we have a valid sketch
+  if (tgtHllType == this->getTgtHllType() && !this->isRebuildKxqCurminFlag()) {
     return static_cast<HllArray*>(copy());
   }
-  if (tgtHllType == target_hll_type::HLL_4) {
-    return HllSketchImplFactory<A>::convertToHll4(*this);
-  } else if (tgtHllType == target_hll_type::HLL_6) {
-    return HllSketchImplFactory<A>::convertToHll6(*this);
-  } else { // tgtHllType == HLL_8
-    return HllSketchImplFactory<A>::convertToHll8(*this);
+  
+  // the factory methods replay the coupons and will always rebuild
+  // the sketch in a consistent way
+  switch (tgtHllType) {
+    case target_hll_type::HLL_4:
+      return HllSketchImplFactory<A>::convertToHll4(*this);
+    case target_hll_type::HLL_6:
+      return HllSketchImplFactory<A>::convertToHll6(*this);
+    case target_hll_type::HLL_8:
+      return HllSketchImplFactory<A>::convertToHll8(*this);
+    default:
+      throw std::invalid_argument("Invalid target HLL type"); 
   }
 }
 
@@ -192,9 +216,9 @@ HllArray<A>* HllArray<A>::newHll(std::istream& is, const A& allocator) {
 }
 
 template<typename A>
-vector_u8<A> HllArray<A>::serialize(bool compact, unsigned header_size_bytes) const {
+auto HllArray<A>::serialize(bool compact, unsigned header_size_bytes) const -> vector_bytes {
   const size_t sketchSizeBytes = (compact ? getCompactSerializationBytes() : getUpdatableSerializationBytes()) + header_size_bytes;
-  vector_u8<A> byteArr(sketchSizeBytes, 0, getAllocator());
+  vector_bytes byteArr(sketchSizeBytes, 0, getAllocator());
   uint8_t* bytes = byteArr.data() + header_size_bytes;
   AuxHashMap<A>* auxHashMap = getAuxHashMap();
 
@@ -299,7 +323,7 @@ double HllArray<A>::getEstimate() const {
   if (oooFlag_) {
     return getCompositeEstimate();
   }
-  return getHipAccum();
+  return hipAccum_;
 }
 
 // HLL UPPER AND LOWER BOUNDS
@@ -322,54 +346,20 @@ double HllArray<A>::getLowerBound(uint8_t numStdDev) const {
   HllUtil<A>::checkNumStdDev(numStdDev);
   const uint32_t configK = 1 << this->lgConfigK_;
   const double numNonZeros = ((curMin_ == 0) ? (configK - numAtCurMin_) : configK);
-
-  double estimate;
-  double rseFactor;
-  if (oooFlag_) {
-    estimate = getCompositeEstimate();
-    rseFactor = hll_constants::HLL_NON_HIP_RSE_FACTOR;
-  } else {
-    estimate = hipAccum_;
-    rseFactor = hll_constants::HLL_HIP_RSE_FACTOR;
-  }
-
-  double relErr;
-  if (this->lgConfigK_ > 12) {
-    relErr = (numStdDev * rseFactor) / sqrt(configK);
-  } else {
-    relErr = HllUtil<A>::getRelErr(false, oooFlag_, this->lgConfigK_, numStdDev);
-  }
-  return fmax(estimate / (1.0 + relErr), numNonZeros);
+  const double relErr = HllUtil<A>::getRelErr(false, this->oooFlag_, this->lgConfigK_, numStdDev);
+  return fmax(getEstimate() / (1.0 + relErr), numNonZeros);
 }
 
 template<typename A>
 double HllArray<A>::getUpperBound(uint8_t numStdDev) const {
   HllUtil<A>::checkNumStdDev(numStdDev);
-  const uint32_t configK = 1 << this->lgConfigK_;
-
-  double estimate;
-  double rseFactor;
-  if (oooFlag_) {
-    estimate = getCompositeEstimate();
-    rseFactor = hll_constants::HLL_NON_HIP_RSE_FACTOR;
-  } else {
-    estimate = hipAccum_;
-    rseFactor = hll_constants::HLL_HIP_RSE_FACTOR;
-  }
-
-  double relErr;
-  if (this->lgConfigK_ > 12) {
-    relErr = (-1.0) * (numStdDev * rseFactor) / sqrt(configK);
-  } else {
-    relErr = HllUtil<A>::getRelErr(true, oooFlag_, this->lgConfigK_, numStdDev);
-  }
-  return estimate / (1.0 + relErr);
+  const double relErr = HllUtil<A>::getRelErr(true, this->oooFlag_, this->lgConfigK_, numStdDev);
+  return getEstimate() / (1.0 + relErr);
 }
 
 /**
  * This is the (non-HIP) estimator.
  * It is called "composite" because multiple estimators are pasted together.
- * @param absHllArr an instance of the AbstractHllArray class.
  * @return the composite estimate
  */
 // Original C: again-two-registers.c hhb_get_composite_estimate L1489
@@ -468,16 +458,6 @@ void HllArray<A>::putNumAtCurMin(uint32_t numAtCurMin) {
   numAtCurMin_ = numAtCurMin;
 }
 
-template<typename A>
-void HllArray<A>::decNumAtCurMin() {
-  --numAtCurMin_;
-}
-
-template<typename A>
-void HllArray<A>::addToHipAccum(double delta) {
-  hipAccum_ += delta;
-}
-
 template<typename A>
 bool HllArray<A>::isCompact() const {
   return false;
@@ -486,7 +466,7 @@ bool HllArray<A>::isCompact() const {
 template<typename A>
 bool HllArray<A>::isEmpty() const {
   const uint32_t configK = 1 << this->lgConfigK_;
-  return (getCurMin() == 0) && (getNumAtCurMin() == configK);
+  return (curMin_ == 0) && (numAtCurMin_ == configK);
 }
 
 template<typename A>
@@ -556,6 +536,11 @@ AuxHashMap<A>* HllArray<A>::getAuxHashMap() const {
   return nullptr;
 }
 
+template<typename A>
+auto HllArray<A>::getHllArray() const -> const vector_bytes& {
+  return hllByteArr_;
+}
+
 template<typename A>
 void HllArray<A>::hipAndKxQIncrementalUpdate(uint8_t oldValue, uint8_t newValue) {
   const uint32_t configK = 1 << this->getLgConfigK();
@@ -601,6 +586,52 @@ double HllArray<A>::getHllRawEstimate() const {
   return hyperEst;
 }
 
+template<typename A>
+void HllArray<A>::setRebuildKxqCurminFlag(bool rebuild) {
+  rebuild_kxq_curmin_ = rebuild;
+}
+
+template<typename A>
+bool HllArray<A>::isRebuildKxqCurminFlag() const {
+  return rebuild_kxq_curmin_;
+}
+
+template<typename A>
+void HllArray<A>::check_rebuild_kxq_cur_min() {
+  if (!rebuild_kxq_curmin_) { return; }
+
+  uint8_t cur_min = 64;
+  uint32_t num_at_cur_min = 0;
+  double kxq0 = 1 << this->lgConfigK_;
+  double kxq1 = 0;
+
+  auto it = this->begin(true); // want all points to adjust cur_min
+  const auto end = this->end();
+  while (it != end) {
+    uint8_t v = HllUtil<A>::getValue(*it);
+    if (v > 0) {
+      if (v < 32) { kxq0 += INVERSE_POWERS_OF_2[v] - 1.0; }
+      else        { kxq1 += INVERSE_POWERS_OF_2[v] - 1.0; }
+    }
+    if (v > cur_min) { ++it; continue; }
+    if (v < cur_min) {
+      cur_min = v;
+      num_at_cur_min = 1;
+    } else {
+      ++num_at_cur_min;
+    }    
+    ++it;
+  }
+
+  kxq0_ = kxq0;
+  kxq1_ = kxq1;
+  curMin_ = cur_min;
+  numAtCurMin_ = num_at_cur_min;
+  rebuild_kxq_curmin_ = false;
+  // HipAccum is not affected
+
+}
+
 template<typename A>
 typename HllArray<A>::const_iterator HllArray<A>::begin(bool all) const {
   return const_iterator(hllByteArr_.data(), 1 << this->lgConfigK_, 0, this->tgtHllType_, nullptr, 0, all);
@@ -637,12 +668,14 @@ bool HllArray<A>::const_iterator::operator!=(const const_iterator& other) const
 }
 
 template<typename A>
-uint32_t HllArray<A>::const_iterator::operator*() const {
+auto HllArray<A>::const_iterator::operator*() const -> reference {
   return HllUtil<A>::pair(index_, value_);
 }
 
 template<typename A>
 uint8_t HllArray<A>::const_iterator::get_value(const uint8_t* array, uint32_t index, target_hll_type hll_type, const AuxHashMap<A>* exceptions, uint8_t offset) {
+  // TODO: we should be able to improve efficiency here by reading multiple bytes at a time
+  //       for HLL4 and HLL6
   if (hll_type == target_hll_type::HLL_4) {
     uint8_t value = array[index >> 1];
     if ((index & 1) > 0) { // odd

data/vendor/datasketches-cpp/hll/include/HllArray.hpp

@@ -31,12 +31,15 @@ class AuxHashMap;
 template<typename A>
 class HllArray : public HllSketchImpl<A> {
   public:
+    using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
+
     HllArray(uint8_t lgConfigK, target_hll_type tgtHllType, bool startFullSize, const A& allocator);
+    explicit HllArray(const HllArray& other, target_hll_type tgtHllType);
 
     static HllArray* newHll(const void* bytes, size_t len, const A& allocator);
     static HllArray* newHll(std::istream& is, const A& allocator);
 
-    virtual vector_u8<A> serialize(bool compact, unsigned header_size_bytes) const;
+    virtual vector_bytes serialize(bool compact, unsigned header_size_bytes) const;
     virtual void serialize(std::ostream& os, bool compact) const;
 
     virtual ~HllArray() = default;
@@ -52,10 +55,6 @@ class HllArray : public HllSketchImpl<A> {
     virtual double getLowerBound(uint8_t numStdDev) const;
     virtual double getUpperBound(uint8_t numStdDev) const;
 
-    inline void addToHipAccum(double delta);
-
-    inline void decNumAtCurMin();
-
     inline uint8_t getCurMin() const;
     inline uint32_t getNumAtCurMin() const;
     inline double getHipAccum() const;
@@ -90,12 +89,18 @@ class HllArray : public HllSketchImpl<A> {
 
     virtual AuxHashMap<A>* getAuxHashMap() const;
 
+    void setRebuildKxqCurminFlag(bool rebuild);
+    bool isRebuildKxqCurminFlag() const;
+    void check_rebuild_kxq_cur_min();
+
     class const_iterator;
     virtual const_iterator begin(bool all = false) const;
     virtual const_iterator end() const;
 
     virtual A getAllocator() const;
 
+    const vector_bytes& getHllArray() const;
+
   protected:
     void hipAndKxQIncrementalUpdate(uint8_t oldValue, uint8_t newValue);
     double getHllBitMapEstimate() const;
@@ -104,21 +109,28 @@ class HllArray : public HllSketchImpl<A> {
     double hipAccum_;
     double kxq0_;
     double kxq1_;
-    vector_u8<A> hllByteArr_; //init by sub-classes
+    vector_bytes hllByteArr_; //init by sub-classes
     uint8_t curMin_; //always zero for Hll6 and Hll8, only tracked by Hll4Array
     uint32_t numAtCurMin_; //interpreted as num zeros when curMin == 0
     bool oooFlag_; //Out-Of-Order Flag
+    bool rebuild_kxq_curmin_; // flag to recompute
 
     friend class HllSketchImplFactory<A>;
 };
 
 template<typename A>
-class HllArray<A>::const_iterator: public std::iterator<std::input_iterator_tag, uint32_t> {
+class HllArray<A>::const_iterator {
 public:
+  using iterator_category = std::input_iterator_tag;
+  using value_type = uint32_t;
+  using difference_type = void;
+  using pointer = uint32_t*;
+  using reference = uint32_t;
+
   const_iterator(const uint8_t* array, uint32_t array_slze, uint32_t index, target_hll_type hll_type, const AuxHashMap<A>* exceptions, uint8_t offset, bool all);
   const_iterator& operator++();
   bool operator!=(const const_iterator& other) const;
-  uint32_t operator*() const;
+  reference operator*() const;
 private:
   const uint8_t* array_;
   uint32_t array_size_;