datasketches 0.5.1 → 0.5.3

data/vendor/datasketches-cpp/sampling/include/ebpps_sample_impl.hpp

@@ -28,6 +28,7 @@
 #include <cmath>
 #include <string>
 #include <sstream>
+#include <type_traits>
 
 namespace datasketches {
 
@@ -365,11 +366,15 @@ std::pair<ebpps_sample<T, A>, size_t> ebpps_sample<T, A>::deserialize(const uint
 
   optional<T> partial_item;
   if (has_partial) {
-    optional<T> tmp; // space to deserialize
-    ptr += sd.deserialize(ptr, end_ptr - ptr, &*tmp, 1);
+    // Space to deserialize.
+    // serde::deserialize expects allocated but not initialized storage.
+    typename std::aligned_storage<sizeof(T), alignof(T)>::type tmp_storage;
+    T* tmp = reinterpret_cast<T*>(&tmp_storage);
+
+    ptr += sd.deserialize(ptr, end_ptr - ptr, tmp, 1);
     // serde did not throw so place item and clean up
-    partial_item.emplace(*tmp);
-    (*tmp).~T();
+    partial_item.emplace(std::move(*tmp));
+    tmp->~T();
   }
 
   return std::pair<ebpps_sample<T,A>, size_t>(
@@ -400,11 +405,15 @@ ebpps_sample<T, A> ebpps_sample<T, A>::deserialize(std::istream& is, const SerDe
 
   optional<T> partial_item;
   if (has_partial) {
-    optional<T> tmp; // space to deserialize
-    sd.deserialize(is, &*tmp, 1);
+    // Space to deserialize.
+    // serde::deserialize expects allocated but not initialized storage.
+    typename std::aligned_storage<sizeof(T), alignof(T)>::type tmp_storage;
+    T* tmp = reinterpret_cast<T*>(&tmp_storage);
+
+    sd.deserialize(is, tmp, 1);
     // serde did not throw so place item and clean up
-    partial_item.emplace(*tmp);
-    (*tmp).~T();
+    partial_item.emplace(std::move(*tmp));
+    tmp->~T();
   }
 
   if (!is.good()) throw std::runtime_error("error reading from std::istream");

data/vendor/datasketches-cpp/sampling/include/ebpps_sketch.hpp

@@ -50,6 +50,11 @@ namespace ebpps_constants {
  * The sample may be smaller than k and the resulting size of the sample potentially includes
  * a probabilistic component, meaning the resulting sample size is not always constant.
  *
+ * Sketch that may retain string values.
+ * For sketches containing strings, cross-language portability depends on
+ * using compatible string encodings. This class does not by itself enforce
+ * UTF-8 validity for all string inputs.
+ *
  * @author Jon Malkin
  */
 template<
@@ -71,6 +76,8 @@ class ebpps_sketch {
     /**
      * Updates this sketch with the given data item with the given weight.
      * This method takes an lvalue.
+     * If cross-language portability is required, callers should ensure that
+     * the input string uses a compatible encoding (valid UTF-8).
      * @param item an item from a stream of items
      * @param weight the weight of the item
      */
@@ -79,6 +86,8 @@ class ebpps_sketch {
     /**
      * Updates this sketch with the given data item with the given weight.
      * This method takes an rvalue.
+     * If cross-language portability is required, callers should ensure that
+     * the input string uses a compatible encoding (valid UTF-8).
      * @param item an item from a stream of items
      * @param weight the weight of the item
      */
@@ -87,6 +96,8 @@ class ebpps_sketch {
     /**
      * Merges the provided sketch into the current one.
      * This method takes an lvalue.
+     * If sketches contain strings, callers are responsible for ensuring that
+     * both sketches were built using compatible string encodings.
      * @param sketch the sketch to merge into the current object
      */
     void merge(const ebpps_sketch<T, A>& sketch);
@@ -94,6 +105,8 @@ class ebpps_sketch {
     /**
      * Merges the provided sketch into the current one.
      * This method takes an rvalue.
+     * If sketches contain strings, callers are responsible for ensuring that
+     * both sketches were built using compatible string encodings.
      * @param sketch the sketch to merge into the current object
      */
     void merge(ebpps_sketch<T, A>&& sketch);

data/vendor/datasketches-cpp/sampling/include/var_opt_sketch.hpp

@@ -57,6 +57,11 @@ namespace var_opt_constants {
  * optimal (varopt) sampling is related to reservoir sampling, with improved error bounds for
  * subset sum estimation.
  *
+ * Sketch that may retain string values.
+ * For sketches containing strings, cross-language portability depends on
+ * using compatible string encodings. This class does not by itself enforce
+ * UTF-8 validity for all string inputs.
+ *
  * author Kevin Lang
  * author Jon Malkin
  */
@@ -111,6 +116,8 @@ class var_opt_sketch {
     /**
      * Updates this sketch with the given data item with the given weight.
      * This method takes an lvalue.
+     * If cross-language portability is required, callers should ensure that
+     * the input string uses a compatible encoding (valid UTF-8).
      * @param item an item from a stream of items
      * @param weight the weight of the item
      */
@@ -119,6 +126,8 @@ class var_opt_sketch {
     /**
      * Updates this sketch with the given data item with the given weight.
      * This method takes an rvalue.
+     * If cross-language portability is required, callers should ensure that
+     * the input string uses a compatible encoding (valid UTF-8).
      * @param item an item from a stream of items
      * @param weight the weight of the item
      */
@@ -263,7 +272,7 @@ class var_opt_sketch {
     typedef typename std::allocator_traits<A>::template rebind_alloc<double> AllocDouble;
     typedef typename std::allocator_traits<A>::template rebind_alloc<bool> AllocBool;
 
-    static const uint32_t MIN_LG_ARR_ITEMS = 3;
+    static const uint32_t MIN_LG_ARR_ITEMS = 4;
 
     static const uint8_t PREAMBLE_LONGS_EMPTY  = 1;
     static const uint8_t PREAMBLE_LONGS_WARMUP = 3;

data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp

@@ -772,12 +772,11 @@ string<A> var_opt_sketch<T, A>::items_to_string(bool print_gap) const {
 template<typename T, typename A>
 template<typename O>
 void var_opt_sketch<T, A>::update(O&& item, double weight, bool mark) {
-  if (weight < 0.0 || std::isnan(weight) || std::isinf(weight)) {
-    throw std::invalid_argument("Item weights must be nonnegative and finite. Found: "
+  if (weight <= 0.0 || std::isnan(weight) || std::isinf(weight)) {
+    throw std::invalid_argument("Item weights must be positive and finite. Found: "
                                 + std::to_string(weight));
-  } else if (weight == 0.0) {
-    return;
   }
+
   ++n_;
 
   if (r_ == 0) {
@@ -1029,7 +1028,7 @@ void var_opt_sketch<T, A>::transition_from_warmup() {
   total_wt_r_ = weights_[k_]; // only one item, known location
   weights_[k_] = -1.0;
 
-  // The two lightest items are ncessarily downsample-able to one item,
+  // The two lightest items are necessarily downsample-able to one item,
   // and are therefore a valid initial candidate set
   grow_candidate_set(weights_[k_ - 1] + total_wt_r_, 2);
 }
@@ -1065,7 +1064,7 @@ void var_opt_sketch<T, A>::restore_towards_leaves(uint32_t slot_in) {
   while (child <= last_slot) {
     uint32_t child2 = child + 1; // might also be invalid
     if ((child2 <= last_slot) && (weights_[child2] < weights_[child])) {
-      // siwtch to other child if it's both valid and smaller
+      // switch to other child if it's both valid and smaller
       child = child2;
     }
 
@@ -1221,7 +1220,7 @@ uint32_t var_opt_sketch<T, A>::choose_delete_slot(double wt_cands, uint32_t num_
     if ((wt_cands * next_double_exclude_zero()) < ((num_cands - 1) * wt_m_cand)) {
       return pick_random_slot_in_r(); // keep item in M
     } else {
-      return h_; // indext of item in M
+      return h_; // index of item in M
     }
   } else {
     // general case

data/vendor/datasketches-cpp/sampling/include/var_opt_union.hpp

@@ -65,13 +65,17 @@ public:
   /**
    * Updates this union with the given sketch
    * This method takes an lvalue.
+   * If sketches contain strings, callers are responsible for ensuring that
+   * both sketches were built using compatible string encodings.
    * @param sk a sketch to add to the union
    */
   void update(const var_opt_sketch<T, A>& sk);
-  
+
   /**
    * Updates this union with the given sketch
    * This method takes an rvalue.
+   * If sketches contain strings, callers are responsible for ensuring that
+   * both sketches were built using compatible string encodings.
    * @param sk a sketch to add to the union
    */
   void update(var_opt_sketch<T, A>&& sk);

data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp

@@ -200,6 +200,7 @@ var_opt_union<T, A> var_opt_union<T, A>::deserialize(const void* bytes, size_t s
     return var_opt_union(max_k);
   }
 
+  ensure_minimum_memory(size, PREAMBLE_LONGS_NON_EMPTY << 3);
   uint64_t items_seen;
   ptr += copy_from_mem(ptr, items_seen);
   double outer_tau_numer;
@@ -590,7 +591,7 @@ void var_opt_union<T, A>::migrate_marked_items_by_decreasing_k(var_opt_sketch<T,
 
   // if non-full and pseudo-exact, change k so that gcopy is full
   if ((r_count == 0) && (h_count < k)) {
-    gcopy.k_ = h_count; // may leve extra space allocated but that's ok
+    gcopy.k_ = h_count; // may leave extra space allocated but that's ok
   }
 
   // Now k equals the number of samples, so reducing k will increase tau.

data/vendor/datasketches-cpp/sampling/test/ebpps_allocation_test.cpp

@@ -63,7 +63,7 @@ TEST_CASE( "ebpps merge", "[ebpps_sketch][test_type]") {
     ebpps_test_sketch sk1(k, 0);
     ebpps_test_sketch sk2(k, 0);
 
-    // move udpates
+    // move updates
     for (int i = 0; i < (int) n; ++i) {
       sk1.update(i);
       sk2.update(-i);

data/vendor/datasketches-cpp/sampling/test/ebpps_sketch_test.cpp

@@ -222,7 +222,7 @@ TEST_CASE("ebpps sketch: merge large into small", "[ebpps_sketch]") {
   sk1 = create_unweighted_sketch(k / 2, 0);
   sk1.update(-1, k / 4.0);
   sk1.update(-2, k / 8.0);
-  // sk2 should have been unchaged
+  // sk2 should have been unchanged
   REQUIRE(sk2.get_n() == k);
   REQUIRE(sk2.get_c() == Approx(k).margin(EPS));
 
@@ -250,7 +250,7 @@ TEST_CASE("ebpps sketch: merge small into large", "[ebpps_sketch]") {
   
   // rvalue merge
   sk1 = create_unweighted_sketch(k, 3 * k / 2);
-  // sk2 should have been unchaged
+  // sk2 should have been unchanged
   REQUIRE(sk2.get_n() == 1);
   REQUIRE(sk2.get_c() == 1.0);
   sk2.update(-2, k / 10.0);

data/vendor/datasketches-cpp/sampling/test/var_opt_allocation_test.cpp

@@ -67,7 +67,7 @@ TEST_CASE( "varopt union: move", "[var_opt_union][test_type]") {
     var_opt_test_sketch sk1(k, var_opt_test_sketch::DEFAULT_RESIZE_FACTOR, 0);
     var_opt_test_sketch sk2(k, var_opt_test_sketch::DEFAULT_RESIZE_FACTOR, 0);
 
-    // move udpates
+    // move updates
     for (int i = 0; i < (int) n; ++i) {
       sk1.update(i);
       sk2.update(-i);

data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp

@@ -178,11 +178,17 @@ TEST_CASE("varopt sketch: non-empty degenerate sketch", "[var_opt_sketch]") {
 
 TEST_CASE("varopt sketch: invalid weight", "[var_opt_sketch]") {
   var_opt_sketch<std::string> sk(100, resize_factor::X2);
-  REQUIRE_THROWS_AS(sk.update("invalid_weight", -1.0), std::invalid_argument);
 
-  // should not throw but sketch should still be empty
-  sk.update("zero weight", 0.0);
-  REQUIRE(sk.is_empty());
+  // Negative
+  REQUIRE_THROWS_AS(sk.update("invalid_weight", -1.0), std::invalid_argument);
+  // Zero
+  REQUIRE_THROWS_AS(sk.update("zero_weight", 0.0), std::invalid_argument);
+  // NaN
+  REQUIRE_THROWS_AS(sk.update("NaN_weight", std::numeric_limits<double>::quiet_NaN()), std::invalid_argument);
+  // +Inf
+  REQUIRE_THROWS_AS(sk.update("positive_infinity", std::numeric_limits<double>::infinity()), std::invalid_argument);
+  // -Inf
+  REQUIRE_THROWS_AS(sk.update("negative_infinity", -std::numeric_limits<double>::infinity()), std::invalid_argument);
 }
 
 TEST_CASE("varopt sketch: corrupt serialized weight", "[var_opt_sketch]") {

data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp

@@ -305,4 +305,16 @@ TEST_CASE("varopt union: serialize sampling", "[var_opt_union]") {
   compare_serialization_deserialization(u);
 }
 
+TEST_CASE("varopt union: deserialize truncated non-empty preamble", "[var_opt_union]") {
+  var_opt_sketch<int> sk(32);
+  sk.update(1);
+  var_opt_union<int> u(32);
+  u.update(sk);
+  auto bytes = u.serialize();
+  // a non-empty union needs 4 preamble longs
+  for (size_t size = 8; size < 32; ++size) {
+    REQUIRE_THROWS_AS(var_opt_union<int>::deserialize(bytes.data(), size), std::out_of_range);
+  }
+}
+
 }

data/vendor/datasketches-cpp/tdigest/include/tdigest.hpp

@@ -108,6 +108,7 @@ public:
 
   /**
    * Update this t-Digest with the given value
+   * NaN and infinity values are ignored
    * @param value to update the t-Digest with
    */
   void update(T value);
@@ -153,6 +154,7 @@ public:
    * Compute approximate normalized rank of the given value.
    *
    * <p>If the sketch is empty this throws std::runtime_error.
+   * <p>NaN value throw std::invalid_argument.
    *
    * @param value to be ranked
    * @return normalized rank (from 0 to 1 inclusive)
@@ -257,16 +259,29 @@ public:
    */
   static tdigest deserialize(const void* bytes, size_t size, const Allocator& allocator = Allocator());
 
+  class const_iterator;
+
+  /**
+   * Iterator pointing to the first centroid in the sketch.
+   * If the sketch is empty, the returned iterator must not be dereferenced or incremented.
+   * @return iterator pointing to the first centroid in the sketch
+   */
+  const_iterator begin() const;
+
+  /**
+   * Iterator pointing to the past-the-end centroid in the sketch.
+   * It does not point to any centroid, and must not be dereferenced or incremented.
+   * @return iterator pointing to the past-the-end centroid in the sketch
+   */
+  const_iterator end() const;
 private:
   bool reverse_merge_;
   uint16_t k_;
-  uint16_t internal_k_;
   T min_;
   T max_;
   size_t centroids_capacity_;
   vector_centroid centroids_;
   uint64_t centroids_weight_;
-  size_t buffer_capacity_;
   vector_t buffer_;
 
   static const size_t BUFFER_MULTIPLIER = 4;
@@ -297,6 +312,27 @@ private:
   static inline void check_split_points(const T* values, uint32_t size);
 };
 
+template<typename T, typename A>
+class tdigest<T, A>::const_iterator {
+public:
+  using iterator_category = std::input_iterator_tag;
+  using value_type = std::pair<T, W>;
+  using difference_type = void;
+  using pointer = const return_value_holder<value_type>;
+  using reference = const value_type;
+
+  const_iterator& operator++();
+  const_iterator& operator++(int);
+  bool operator==(const const_iterator& other) const;
+  bool operator!=(const const_iterator& other) const;
+  reference operator*() const;
+  pointer operator->() const;
+private:
+  friend class tdigest;
+  uint32_t index_;
+  vector_centroid centroids_;
+  const_iterator(const tdigest& tdigest_, bool is_end);
+};
 } /* namespace datasketches */
 
 #include "tdigest_impl.hpp"