datasketches 0.3.1 → 0.3.2

data/vendor/datasketches-cpp/sampling/include/var_opt_sketch.hpp

@@ -346,14 +346,21 @@ class var_opt_sketch {
 };
 
 template<typename T, typename A>
-class var_opt_sketch<T, A>::const_iterator : public std::iterator<std::input_iterator_tag, T> {
+class var_opt_sketch<T, A>::const_iterator {
 public:
+  using iterator_category = std::input_iterator_tag;
+  using value_type = std::pair<const T&, const double>;
+  using difference_type = void;
+  using pointer = const return_value_holder<value_type>;
+  using reference = const value_type;
+
   const_iterator(const const_iterator& other);
   const_iterator& operator++();
   const_iterator& operator++(int);
   bool operator==(const const_iterator& other) const;
   bool operator!=(const const_iterator& other) const;
-  const std::pair<const T&, const double> operator*() const;
+  reference operator*() const;
+  pointer operator->() const;
 
 private:
   friend class var_opt_sketch<T, A>;
@@ -362,8 +369,8 @@ private:
   // default iterator over full sketch
   const_iterator(const var_opt_sketch<T, A>& sk, bool is_end);
   
-  // iterates over only one of the H or R region, optionally applying weight correction
-  // to R region (can correct for numerical precision issues)
+  // iterates over only one of the H or R regions
+  // does not apply weight correction
   const_iterator(const var_opt_sketch<T, A>& sk, bool is_end, bool use_r_region);
 
   bool get_mark() const;
@@ -377,14 +384,21 @@ private:
 
 // non-const iterator for internal use
 template<typename T, typename A>
-class var_opt_sketch<T, A>::iterator : public std::iterator<std::input_iterator_tag, T> {
+class var_opt_sketch<T, A>::iterator {
 public:
+  using iterator_category = std::input_iterator_tag;
+  using value_type = std::pair<T&, double>;
+  using difference_type = void;
+  using pointer = return_value_holder<value_type>;
+  using reference = value_type;
+
   iterator(const iterator& other);
   iterator& operator++();
   iterator& operator++(int);
   bool operator==(const iterator& other) const;
   bool operator!=(const iterator& other) const;
-  std::pair<T&, double> operator*();
+  reference operator*();
+  pointer operator->();
 
 private:
   friend class var_opt_sketch<T, A>;

data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp

@@ -189,16 +189,16 @@ var_opt_sketch<T, A>::~var_opt_sketch() {
       // destroy everything
       const size_t num_to_destroy = std::min(k_ + 1, curr_items_alloc_);
       for (size_t i = 0; i < num_to_destroy; ++i) {
-        allocator_.destroy(data_ + i);
+        data_[i].~T();
       }
     } else {
       // skip gap or anything unused at the end
       for (size_t i = 0; i < h_; ++i) {
-        allocator_.destroy(data_+ i);
+        data_[i].~T();
       }
     
       for (size_t i = h_ + 1; i < h_ + r_ + 1; ++i) {
-        allocator_.destroy(data_ + i);
+        data_[i].~T();
       }
     }
     allocator_.deallocate(data_, curr_items_alloc_);
@@ -658,14 +658,14 @@ void var_opt_sketch<T, A>::reset() {
     // destroy everything
     const size_t num_to_destroy = std::min(k_ + 1, prev_alloc);
     for (size_t i = 0; i < num_to_destroy; ++i) 
-      allocator_.destroy(data_ + i);
+      data_[i].~T();
   } else {
     // skip gap or anything unused at the end
     for (size_t i = 0; i < h_; ++i)
-      allocator_.destroy(data_+ i);
+      data_[i].~T();
     
     for (size_t i = h_ + 1; i < h_ + r_ + 1; ++i)
-      allocator_.destroy(data_ + i);
+      data_[i].~T();
   }
 
   if (curr_items_alloc_ < prev_alloc) {
@@ -754,10 +754,10 @@ string<A> var_opt_sketch<T, A>::items_to_string(bool print_gap) const {
   const uint32_t array_length = (n_ < k_ ? n_ : k_ + 1);
   for (uint32_t i = 0, display_idx = 0; i < array_length; ++i) {
     if (i == h_ && print_gap) {
-      os << i << ": GAP" << std::endl;
+      os << display_idx << ": GAP" << std::endl;
       ++display_idx;
     } else {
-      os << i << ": " << data_[i] << "\twt = ";
+      os << display_idx << ": " << data_[i] << "\twt = ";
       if (weights_[i] == -1.0) {
         os << get_tau() << "\t(-1.0)" << std::endl;
       } else {
@@ -990,7 +990,7 @@ void var_opt_sketch<T, A>::grow_data_arrays() {
 
     for (uint32_t i = 0; i < prev_size; ++i) {
       new (&tmp_data[i]) T(std::move(data_[i]));
-      allocator_.destroy(data_ + i);
+      data_[i].~T();
       tmp_weights[i] = weights_[i];
     }
 
@@ -1531,7 +1531,6 @@ var_opt_sketch<T, A>::const_iterator::const_iterator(const var_opt_sketch& sk, b
   if (idx_ == final_idx_) { sk_ = nullptr; }
 }
 
-
 template<typename T, typename A>
 var_opt_sketch<T, A>::const_iterator::const_iterator(const const_iterator& other) :
   sk_(other.sk_),
@@ -1543,6 +1542,9 @@ var_opt_sketch<T, A>::const_iterator::const_iterator(const const_iterator& other
 
 template<typename T, typename A>
 typename var_opt_sketch<T, A>::const_iterator& var_opt_sketch<T, A>::const_iterator::operator++() {
+  // accumulate weight already visited
+  if (idx_ > sk_->h_) { cum_r_weight_ += r_item_wt_; }
+
   ++idx_;
   
   if (idx_ == final_idx_) {
@@ -1551,7 +1553,6 @@ typename var_opt_sketch<T, A>::const_iterator& var_opt_sketch<T, A>::const_itera
   } else if (idx_ == sk_->h_ && sk_->r_ > 0) { // check for the gap
     ++idx_;
   }
-  if (idx_ > sk_->h_) { cum_r_weight_ += r_item_wt_; }
   return *this;
 }
 
@@ -1575,14 +1576,19 @@ bool var_opt_sketch<T, A>::const_iterator::operator!=(const const_iterator& othe
 }
 
 template<typename T, typename A>
-const std::pair<const T&, const double> var_opt_sketch<T, A>::const_iterator::operator*() const {
+auto var_opt_sketch<T, A>::const_iterator::operator*() const -> reference {
   double wt;
   if (idx_ < sk_->h_) {
     wt = sk_->weights_[idx_];
   } else {
     wt = r_item_wt_;
   }
-  return std::pair<const T&, const double>(sk_->data_[idx_], wt);
+  return value_type(sk_->data_[idx_], wt);
+}
+
+template<typename T, typename A>
+auto var_opt_sketch<T, A>::const_iterator::operator->() const -> pointer {
+  return **this;
 }
 
 template<typename T, typename A>
@@ -1622,6 +1628,9 @@ var_opt_sketch<T, A>::iterator::iterator(const iterator& other) :
 
 template<typename T, typename A>
 typename var_opt_sketch<T, A>::iterator& var_opt_sketch<T, A>::iterator::operator++() {
+  // accumulate weight already visited
+  if (idx_ > sk_->h_) { cum_r_weight_ += r_item_wt_; }
+
   ++idx_;
   
   if (idx_ == final_idx_) {
@@ -1630,7 +1639,7 @@ typename var_opt_sketch<T, A>::iterator& var_opt_sketch<T, A>::iterator::operato
   } else if (idx_ == sk_->h_ && sk_->r_ > 0) { // check for the gap
     ++idx_;
   }
-  if (idx_ > sk_->h_) { cum_r_weight_ += r_item_wt_; }
+
   return *this;
 }
 
@@ -1654,7 +1663,7 @@ bool var_opt_sketch<T, A>::iterator::operator!=(const iterator& other) const {
 }
 
 template<typename T, typename A>
-std::pair<T&, double> var_opt_sketch<T, A>::iterator::operator*() {
+auto var_opt_sketch<T, A>::iterator::operator*() -> reference {
   double wt;
   if (idx_ < sk_->h_) {
     wt = sk_->weights_[idx_];
@@ -1663,7 +1672,12 @@ std::pair<T&, double> var_opt_sketch<T, A>::iterator::operator*() {
   } else {
     wt = r_item_wt_;
   }
-  return std::pair<T&, double>(sk_->data_[idx_], wt);
+  return value_type(sk_->data_[idx_], wt);
+}
+
+template<typename T, typename A>
+auto var_opt_sketch<T, A>::iterator::operator->() -> pointer {
+  return **this;
 }
 
 template<typename T, typename A>

data/vendor/datasketches-cpp/sampling/include/var_opt_union.hpp

@@ -153,6 +153,8 @@ public:
 
 private:
   typedef typename std::allocator_traits<A>::template rebind_alloc<var_opt_sketch<T, A>> AllocSketch;
+  typedef typename std::allocator_traits<A>::template rebind_alloc<double> AllocDouble;
+  typedef typename std::allocator_traits<A>::template rebind_alloc<bool> AllocBool;
 
   static const uint8_t PREAMBLE_LONGS_EMPTY = 1;
   static const uint8_t PREAMBLE_LONGS_NON_EMPTY = 4;
@@ -170,10 +172,12 @@ private:
 
   uint32_t max_k_;
 
+  A allocator_;
+
   var_opt_sketch<T, A> gadget_;
 
   var_opt_union(uint64_t n, double outer_tau_numer, uint64_t outer_tau_denom,
-                uint32_t max_k, var_opt_sketch<T, A>&& gadget);
+                uint32_t max_k, var_opt_sketch<T, A>&& gadget, const A& allocator = A());
 
   /*
    IMPORTANT NOTE: the "gadget" in the union object appears to be a varopt sketch,

data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp

@@ -34,6 +34,7 @@ var_opt_union<T, A>::var_opt_union(uint32_t max_k, const A& allocator) :
   outer_tau_numer_(0.0),
   outer_tau_denom_(0),
   max_k_(max_k),
+  allocator_(allocator),
   gadget_(max_k, var_opt_sketch<T, A>::DEFAULT_RESIZE_FACTOR, true, allocator)
 {}
 
@@ -43,6 +44,7 @@ var_opt_union<T, A>::var_opt_union(const var_opt_union& other) :
   outer_tau_numer_(other.outer_tau_numer_),
   outer_tau_denom_(other.outer_tau_denom_),
   max_k_(other.max_k_),
+  allocator_(other.allocator_),
   gadget_(other.gadget_)
 {}
 
@@ -52,16 +54,18 @@ var_opt_union<T, A>::var_opt_union(var_opt_union&& other) noexcept :
   outer_tau_numer_(other.outer_tau_numer_),
   outer_tau_denom_(other.outer_tau_denom_),
   max_k_(other.max_k_),
+  allocator_(other.allocator_),
   gadget_(std::move(other.gadget_))
 {}
 
 template<typename T, typename A>
 var_opt_union<T, A>::var_opt_union(uint64_t n, double outer_tau_numer, uint64_t outer_tau_denom,
-                                    uint32_t max_k, var_opt_sketch<T, A>&& gadget) :
+                                    uint32_t max_k, var_opt_sketch<T, A>&& gadget, const A& allocator) :
   n_(n),
   outer_tau_numer_(outer_tau_numer),
   outer_tau_denom_(outer_tau_denom),
   max_k_(max_k),
+  allocator_(allocator),
   gadget_(gadget)
 {}
 
@@ -75,6 +79,7 @@ var_opt_union<T, A>& var_opt_union<T, A>::operator=(const var_opt_union& other)
   std::swap(outer_tau_numer_, union_copy.outer_tau_numer_);
   std::swap(outer_tau_denom_, union_copy.outer_tau_denom_);
   std::swap(max_k_, union_copy.max_k_);
+  std::swap(allocator_, other.allocator_);
   std::swap(gadget_, union_copy.gadget_);
   return *this;
 }
@@ -85,6 +90,7 @@ var_opt_union<T, A>& var_opt_union<T, A>::operator=(var_opt_union&& other) {
   std::swap(outer_tau_numer_, other.outer_tau_numer_);
   std::swap(outer_tau_denom_, other.outer_tau_denom_);
   std::swap(max_k_, other.max_k_);
+  std::swap(allocator_, other.allocator_);
   std::swap(gadget_, other.gadget_);
   return *this;
 }
@@ -162,7 +168,7 @@ var_opt_union<T, A> var_opt_union<T, A>::deserialize(std::istream& is, const Ser
   if (!is.good())
     throw std::runtime_error("error reading from std::istream"); 
 
-  return var_opt_union(items_seen, outer_tau_numer, outer_tau_denom, max_k, std::move(gadget));
+  return var_opt_union(items_seen, outer_tau_numer, outer_tau_denom, max_k, std::move(gadget), allocator);
 }
 
 template<typename T, typename A>
@@ -204,7 +210,7 @@ var_opt_union<T, A> var_opt_union<T, A>::deserialize(const void* bytes, size_t s
   const size_t gadget_size = size - (PREAMBLE_LONGS_NON_EMPTY << 3);
   var_opt_sketch<T, A> gadget = var_opt_sketch<T, A>::deserialize(ptr, gadget_size, sd, allocator);
 
-  return var_opt_union(items_seen, outer_tau_numer, outer_tau_denom, max_k, std::move(gadget));
+  return var_opt_union(items_seen, outer_tau_numer, outer_tau_denom, max_k, std::move(gadget), allocator);
 }
 
 template<typename T, typename A>
@@ -508,9 +514,8 @@ void var_opt_union<T, A>::mark_moving_gadget_coercer(var_opt_sketch<T, A>& sk) c
   uint32_t result_r = 0;
   size_t next_r_pos = result_k; // = (result_k+1)-1, to fill R region from back to front
 
-  typedef typename std::allocator_traits<A>::template rebind_alloc<double> AllocDouble;
-  double* wts = AllocDouble().allocate(result_k + 1);
-  T* data     = A().allocate(result_k + 1);
+  double* wts = AllocDouble(allocator_).allocate(result_k + 1);
+  T* data     = A(allocator_).allocate(result_k + 1);
     
   // insert R region items, ignoring weights
   // Currently (May 2017) this next block is unreachable; this coercer is used only in the
@@ -519,7 +524,7 @@ void var_opt_union<T, A>::mark_moving_gadget_coercer(var_opt_sketch<T, A>& sk) c
   // Addedndum (Jan 2020): Cleanup at end of method assumes R count is 0
   const size_t final_idx = gadget_.get_num_samples();
   for (size_t idx = gadget_.h_ + 1; idx <= final_idx; ++idx) {
-    A().construct(&data[next_r_pos], T(gadget_.data_[idx]));
+    new (&data[next_r_pos]) T(gadget_.data_[idx]);
     wts[next_r_pos]  = gadget_.weights_[idx];
     ++result_r;
     --next_r_pos;
@@ -530,20 +535,20 @@ void var_opt_union<T, A>::mark_moving_gadget_coercer(var_opt_sketch<T, A>& sk) c
   // insert H region items
   for (size_t idx = 0; idx < gadget_.h_; ++idx) {
     if (gadget_.marks_[idx]) {
-      A().construct(&data[next_r_pos], T(gadget_.data_[idx]));
+      new (&data[next_r_pos]) T(gadget_.data_[idx]);
       wts[next_r_pos] = -1.0;
       transferred_weight += gadget_.weights_[idx];
       ++result_r;
       --next_r_pos;
     } else {
-      A().construct(&data[result_h], T(gadget_.data_[idx]));
+      new (&data[result_h]) T(gadget_.data_[idx]);
       wts[result_h] = gadget_.weights_[idx];
       ++result_h;
     }
   }
 
   if (result_h + result_r != result_k) throw std::logic_error("H + R counts must equal k");
-  if (fabs(transferred_weight - outer_tau_numer_) > 1e-10) {
+  if (std::abs(transferred_weight - outer_tau_numer_) > 1e-10) {
     throw std::logic_error("uexpected mismatch in transferred weight");
   }
 
@@ -554,11 +559,10 @@ void var_opt_union<T, A>::mark_moving_gadget_coercer(var_opt_sketch<T, A>& sk) c
   wts[result_h] = -1.0;
 
   // clean up arrays in input sketch, replace with new values
-  typedef typename std::allocator_traits<A>::template rebind_alloc<bool> AllocBool;
-  AllocBool().deallocate(sk.marks_, sk.curr_items_alloc_);
-  AllocDouble().deallocate(sk.weights_, sk.curr_items_alloc_);
-  for (size_t i = 0; i < result_k; ++i) { A().destroy(sk.data_ + i); } // assumes everything in H region, no gap
-  A().deallocate(sk.data_, sk.curr_items_alloc_);
+  AllocBool(allocator_).deallocate(sk.marks_, sk.curr_items_alloc_);
+  AllocDouble(allocator_).deallocate(sk.weights_, sk.curr_items_alloc_);
+  for (size_t i = 0; i < result_k; ++i) { sk.data_[i].~T(); } // assumes everything in H region, no gap
+  A(allocator_).deallocate(sk.data_, sk.curr_items_alloc_);
 
   sk.data_ = data;
   sk.weights_ = wts;

data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp

@@ -52,17 +52,15 @@ static void check_if_equal(var_opt_sketch<T, A>& sk1, var_opt_sketch<T, A>& sk2)
   REQUIRE(sk1.get_k() == sk2.get_k());
   REQUIRE(sk1.get_n() == sk2.get_n());
   REQUIRE(sk1.get_num_samples() == sk2.get_num_samples());
-      
+
   auto it1 = sk1.begin();
   auto it2 = sk2.begin();
-  size_t i = 0;
 
   while ((it1 != sk1.end()) && (it2 != sk2.end())) {
-    const std::pair<const T&, const double> p1 = *it1;
-    const std::pair<const T&, const double> p2 = *it2;
+    auto p1 = *it1;
+    auto p2 = *it2;
     REQUIRE(p1.first == p2.first);   // data values
     REQUIRE(p1.second == p2.second); // weights
-    ++i;
     ++it1;
     ++it2;
   }
@@ -182,7 +180,7 @@ TEST_CASE("varopt sketch: invalid weight", "[var_opt_sketch]") {
   var_opt_sketch<std::string> sk(100, resize_factor::X2);
   REQUIRE_THROWS_AS(sk.update("invalid_weight", -1.0), std::invalid_argument);
 
-  // should not throw but sketch shoulds till be empty
+  // should not throw but sketch should still be empty
   sk.update("zero weight", 0.0);
   REQUIRE(sk.is_empty());
 }
@@ -213,7 +211,7 @@ TEST_CASE("varopt sketch: cumulative weight", "[var_opt_sketch]") {
 
   double input_sum = 0.0;
   for (size_t i = 0; i < n; ++i) {
-    // generate weights aboev and below 1.0 using w ~ exp(5*N(0,1))
+    // generate weights above and below 1.0 using w ~ exp(5*N(0,1))
     // which covers about 10 orders of magnitude
     double w = std::exp(5 * N(rand));
     input_sum += w;
@@ -221,12 +219,12 @@ TEST_CASE("varopt sketch: cumulative weight", "[var_opt_sketch]") {
   }
 
   double output_sum = 0.0;
-  for (auto it : sk) { // std::pair<int, weight>
-    output_sum += it.second;
+  for (auto pair : sk) { // std::pair<int, weight>
+    output_sum += pair.second;
   }
     
   double weight_ratio = output_sum / input_sum;
-  REQUIRE(std::abs(weight_ratio - 1.0) == Approx(0).margin(EPS));
+  REQUIRE(weight_ratio == Approx(1.0).margin(EPS));
 }
 
 TEST_CASE("varopt sketch: under-full sketch serialization", "[var_opt_sketch]") {
@@ -275,26 +273,38 @@ TEST_CASE("varopt sketch: full sketch serialization", "[var_opt_sketch]") {
   sk.update(100, 100.0);
   sk.update(101, 101.0);
 
+  subset_summary summary = sk.estimate_subset_sum([](int){ return true; });
+  double total_weight = summary.total_sketch_weight;
+  double cum_weight = 0.0;
+  for (auto pair : sk) {
+    cum_weight += pair.second;
+  }
+  double weight_ratio = cum_weight / total_weight;
+  REQUIRE(weight_ratio == Approx(1.0).margin(EPS));
+
   // first 2 entries should be heavy and in heap order (smallest at root)
   auto it = sk.begin();
-  const std::pair<const int, const double> p1 = *it;
+  auto p1 = *it;
   ++it;
-  const std::pair<const int, const double> p2 = *it;
+  auto p2 = *it;
   REQUIRE(p1.second == Approx(100.0).margin(EPS));
   REQUIRE(p2.second == Approx(101.0).margin(EPS));
   REQUIRE(p1.first == 100);
   REQUIRE(p2.first == 101);
+  // using operator ->
+  REQUIRE(it->first == p2.first);
+  REQUIRE(it->second == p2.second);
 
   // check for 4 preamble longs
   auto bytes = sk.serialize();
   REQUIRE((bytes.data()[0] & 0x3f) == 4);; // PREAMBLE_LONGS_WARMUP
 
-  var_opt_sketch<int> sk_from_bytes = var_opt_sketch<int>::deserialize(bytes.data(), bytes.size());
+  auto sk_from_bytes = var_opt_sketch<int>::deserialize(bytes.data(), bytes.size());
   check_if_equal(sk, sk_from_bytes);
 
   std::stringstream ss(std::ios::in | std::ios::out | std::ios::binary);
   sk.serialize(ss);
-  var_opt_sketch<int> sk_from_stream = var_opt_sketch<int>::deserialize(ss);
+  auto sk_from_stream = var_opt_sketch<int>::deserialize(ss);
   check_if_equal(sk, sk_from_stream);
 
   // ensure we unroll properly
@@ -340,6 +350,15 @@ TEST_CASE("varopt sketch: pseudo-light update", "[var_opt_sketch]") {
   auto it = sk.begin();
   double wt = (*it).second;
   REQUIRE(wt == Approx((k + 2.0) / k).margin(EPS));
+
+  subset_summary summary = sk.estimate_subset_sum([](int){ return true; });
+  double total_weight = summary.total_sketch_weight;
+  double cum_weight = 0.0;
+  for (auto pair : sk) {
+    cum_weight += pair.second;
+  }
+  double weight_ratio = cum_weight / total_weight;
+  REQUIRE(weight_ratio == Approx(1.0).margin(EPS));
 }
 
 TEST_CASE("varopt sketch: pseudo-heavy update", "[var_opt_sketch]") {

data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp

@@ -57,7 +57,6 @@ static void check_if_equal(var_opt_sketch<T, A>& sk1, var_opt_sketch<T, A>& sk2,
       
   auto it1 = sk1.begin();
   auto it2 = sk2.begin();
-  size_t i = 0;
 
   while ((it1 != sk1.end()) && (it2 != sk2.end())) {
     const std::pair<const T&, const double> p1 = *it1;
@@ -66,7 +65,6 @@ static void check_if_equal(var_opt_sketch<T, A>& sk1, var_opt_sketch<T, A>& sk2,
       REQUIRE(p1.first == p2.first); // data values
     }
     REQUIRE(p1.second == p2.second); // weight values
-    ++i;
     ++it1;
     ++it2;
   }

data/vendor/datasketches-cpp/setup.py

@@ -100,7 +100,7 @@ setup(
     url='http://datasketches.apache.org',
     long_description=open('python/README.md').read(),
     long_description_content_type='text/markdown',
-    packages=find_packages(where='python',exclude=['src','*tests*']), # src not needed if only the .so
+    packages=find_packages(where='python',exclude=['src','include','*tests*']), # src not needed if only the .so
     package_dir={'':'python'},
     # may need to add all source paths for sdist packages w/o MANIFEST.in
     ext_modules=[CMakeExtension('datasketches')],

data/vendor/datasketches-cpp/theta/CMakeLists.txt

@@ -62,4 +62,5 @@ install(FILES
 			include/bounds_on_ratios_in_theta_sketched_sets.hpp
 			include/compact_theta_sketch_parser.hpp
 			include/compact_theta_sketch_parser_impl.hpp
+			include/bit_packing.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")