datasketches 0.3.2 → 0.4.0

data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_serialize_for_java.cpp

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <quantiles_sketch.hpp>
+
+namespace datasketches {
+
+TEST_CASE("quantiles sketch double generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    quantiles_sketch<double> sketch;
+    for (unsigned i = 1; i <= n; ++i) sketch.update(i);
+    std::ofstream os("quantiles_double_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.serialize(os);
+  }
+}
+
+struct compare_as_number {
+  bool operator()(const std::string& a, const std::string& b) const {
+    return std::stoi(a) < std::stoi(b);
+  }
+};
+
+TEST_CASE("quantiles sketch string generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    quantiles_sketch<std::string, compare_as_number> sketch;
+    for (unsigned i = 1; i <= n; ++i) sketch.update(std::to_string(i));
+    std::ofstream os("quantiles_string_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.serialize(os);
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_test.cpp

@@ -65,8 +65,6 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
     REQUIRE_THROWS_AS(sketch.get_max_item(), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_rank(0), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_quantile(0.5), std::runtime_error);
-    const double fractions[3] {0, 0.5, 1};
-    REQUIRE_THROWS_AS(sketch.get_quantiles(fractions, 3).empty(), std::runtime_error);
     const float split_points[1] {0};
     REQUIRE_THROWS_AS(sketch.get_PMF(split_points, 1), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_CDF(split_points, 1), std::runtime_error);
@@ -98,13 +96,6 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
     REQUIRE(sketch.get_max_item() == 1.0);
     REQUIRE(sketch.get_quantile(0.5) == 1.0);
 
-    const double fractions[3] {0, 0.5, 1};
-    auto quantiles = sketch.get_quantiles(fractions, 3);
-    REQUIRE(quantiles.size() == 3);
-    REQUIRE(quantiles[0] == 1.0);
-    REQUIRE(quantiles[1] == 1.0);
-    REQUIRE(quantiles[2] == 1.0);
-
     int count = 0;
     for (auto pair: sketch) {
       REQUIRE(pair.second == 1);
@@ -153,20 +144,6 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
     REQUIRE(sketch.get_max_item() == n);
     REQUIRE(sketch.get_quantile(1) == n);
 
-    const double ranks[3] {0, 0.5, 1};
-    auto quantiles = sketch.get_quantiles(ranks, 3);
-    REQUIRE(quantiles.size() == 3);
-    REQUIRE(quantiles[0] == 1);
-    REQUIRE(quantiles[1] == static_cast<float>(n / 2));
-    REQUIRE(quantiles[2] == n);
-
-    // the alternative method must produce the same result
-    auto quantiles2 = sketch.get_quantiles(3);
-    REQUIRE(quantiles2.size() == 3);
-    REQUIRE(quantiles[0] == quantiles2[0]);
-    REQUIRE(quantiles[1] == quantiles2[1]);
-    REQUIRE(quantiles[2] == quantiles2[2]);
-
     int count = 0;
     for (auto pair: sketch) {
       REQUIRE(pair.second == 1);
@@ -283,16 +260,16 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
     // value pushed into higher level
     REQUIRE(sketch.get_rank(50, false) == Approx(0.49).margin(0.01));
     REQUIRE(sketch.get_rank(50, true) == 0.50);
-  
+
     // get_quantile()
     // value still in base buffer
     REQUIRE(sketch.get_quantile(0.70, false) == 71);
     REQUIRE(sketch.get_quantile(0.70, true) == 70);
-  
+
     // value pushed into higher levell
     int quantile = sketch.get_quantile(0.30, false);
     if (quantile != 31 && quantile != 32) { FAIL(); }
-    
+
     quantile = sketch.get_quantile(0.30, true);
     if (quantile != 29 && quantile != 30) { FAIL(); }
   }
@@ -550,7 +527,6 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
     REQUIRE(sketch1.get_quantile(0.5) == Approx(n).margin(n * RANK_EPS_FOR_K_128));
   }
 
-
   SECTION("merge lower k") {
     quantiles_float_sketch sketch1(256, std::less<float>(), 0);
     quantiles_float_sketch sketch2(128, std::less<float>(), 0);
@@ -653,7 +629,7 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
     for (int i = 0; i < 100 * k; i++) {
       sketch1.update(static_cast<float>(i));
     }
-    
+
     sketch1.merge(sketch2);
     REQUIRE(sketch1.get_n() == 101 * k);
     REQUIRE(sketch1.get_k() == 2 * k); // no reason to have shrunk
@@ -670,7 +646,7 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
       sketch1.update(static_cast<float>(i));
       sketch2.update(static_cast<float>(i));
     }
-    
+
     for (int i = 0; i < 100 * k; i++) {
       sketch2.update(static_cast<float>(i));
     }
@@ -691,7 +667,7 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
       sketch1.update(static_cast<float>(i));
       sketch2.update(static_cast<float>(-i));
     }
-    
+
     sketch1.merge(sketch2);
     REQUIRE(sketch1.get_n() == 200 * k);
     REQUIRE(sketch1.get_k() == k); // no reason to have shrunk
@@ -709,7 +685,7 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
       sketch1.update(static_cast<float>(i));
       sketch2.update(static_cast<float>(k - i - 1));
     }
-    
+
     for (int i = k; i < 100 * k; i++) {
       sketch2.update(static_cast<float>(i));
     }
@@ -733,7 +709,7 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
       sketch1.update(static_cast<float>(i));
       sketch2.update(static_cast<float>(2 * n - i - 1));
     }
-    
+
     sketch1.merge(sketch2);
     REQUIRE(sketch1.get_n() == 2 * n);
     REQUIRE(sketch1.get_k() == k);
@@ -752,7 +728,7 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
       sketch1.update(static_cast<float>(i));
       sketch2.update(static_cast<float>(2 * n - i - 1));
     }
-    
+
     sketch1.merge(sketch2);
     REQUIRE(sketch1.get_n() == 2 * n);
     REQUIRE(sketch1.get_k() == k);
@@ -909,10 +885,10 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
     const uint16_t k = 8;
     const int n = 403;
     quantiles_sketch<double> sk_double(k);
-    
+
     quantiles_sketch<float> sk_float(k);
     REQUIRE(sk_float.is_empty());
-    
+
     for (int i = 0; i < n; ++i) sk_double.update(i + .01);
 
     quantiles_sketch<int> sk_int(sk_double);
@@ -921,13 +897,13 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
     REQUIRE(sk_double.get_num_retained() == sk_int.get_num_retained());
 
     auto sv_double = sk_double.get_sorted_view();
-    std::vector<std::pair<double, uint64_t>> vec_double(sv_double.begin(), sv_double.end()); 
+    std::vector<std::pair<double, uint64_t>> vec_double(sv_double.begin(), sv_double.end());
 
     auto sv_int = sk_int.get_sorted_view();
-    std::vector<std::pair<int, uint64_t>> vec_int(sv_int.begin(), sv_int.end()); 
+    std::vector<std::pair<int, uint64_t>> vec_int(sv_int.begin(), sv_int.end());
 
     REQUIRE(vec_double.size() == vec_int.size());
-    
+
     for (size_t i = 0; i < vec_int.size(); ++i) {
       // known truncation with conversion so approximate result
       REQUIRE(vec_double[i].first == Approx(vec_int[i].first).margin(0.1));

data/vendor/datasketches-cpp/req/include/req_common.hpp

@@ -27,10 +27,14 @@
 
 namespace datasketches {
 
+/// REQ sketch constants
 namespace req_constants {
-  static const uint16_t MIN_K = 4;
-  static const uint8_t INIT_NUM_SECTIONS = 3;
-  static const unsigned MULTIPLIER = 2;
+  /// minimum value of parameter K
+  const uint16_t MIN_K = 4;
+  /// initial number of sections
+  const uint8_t INIT_NUM_SECTIONS = 3;
+  /// multiplier for nominal capacity
+  const unsigned MULTIPLIER = 2;
 }
 
 } /* namespace datasketches */

data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp

@@ -277,7 +277,7 @@ std::pair<uint32_t, uint32_t> req_compactor<T, C, A>::compact(req_compactor& nex
   if (compaction_range.second - compaction_range.first < 2) throw std::logic_error("compaction range error");
 
   if ((state_ & 1) == 1) { coin_ = !coin_; } // for odd flip coin;
-  else { coin_ = random_bit(); } // random coin flip
+  else { coin_ = random_utils::random_bit(); } // random coin flip
 
   const auto num = (compaction_range.second - compaction_range.first) / 2;
   next.ensure_space(num);
@@ -493,7 +493,7 @@ comparator_(comparator),
 allocator_(allocator),
 lg_weight_(lg_weight),
 hra_(hra),
-coin_(random_bit()),
+coin_(random_utils::random_bit()),
 sorted_(sorted),
 section_size_raw_(section_size_raw),
 section_size_(nearest_even(section_size_raw)),

data/vendor/datasketches-cpp/req/include/req_sketch.hpp

@@ -25,9 +25,52 @@
 #include "req_common.hpp"
 #include "req_compactor.hpp"
 #include "quantiles_sorted_view.hpp"
+#include "optional.hpp"
 
 namespace datasketches {
 
+/**
+ * Relative Error Quantiles Sketch.
+ * This is an implementation based on the paper
+ * "Relative Error Streaming Quantiles" by Graham Cormode, Zohar Karnin, Edo Liberty,
+ * Justin Thaler, Pavel Veselý, and loosely derived from a Python prototype written by Pavel Veselý.
+ *
+ * <p>Reference: https://arxiv.org/abs/2004.01668</p>
+ *
+ * <p>This implementation differs from the algorithm described in the paper in the following:</p>
+ *
+ * <ul>
+ * <li>The algorithm requires no upper bound on the stream length.
+ * Instead, each relative-compactor counts the number of compaction operations performed
+ * so far (via variable state). Initially, the relative-compactor starts with INIT_NUMBER_OF_SECTIONS.
+ * Each time the number of compactions (variable state) exceeds 2^{numSections - 1}, we double
+ * numSections. Note that after merging the sketch with another one variable state may not correspond
+ * to the number of compactions performed at a particular level, however, since the state variable
+ * never exceeds the number of compactions, the guarantees of the sketch remain valid.</li>
+ *
+ * <li>The size of each section (variable k and section_size in the code and parameter k in
+ * the paper) is initialized with a number set by the user via variable k.
+ * When the number of sections doubles, we decrease section_size by a factor of sqrt(2).
+ * This is applied at each level separately. Thus, when we double the number of sections, the
+ * nominal compactor size increases by a factor of approx. sqrt(2) (+/- rounding).</li>
+ *
+ * <li>The merge operation here does not perform "special compactions", which are used in the paper
+ * to allow for a tight mathematical analysis of the sketch.</li>
+ * </ul>
+ *
+ * <p>This implementation provides a number of capabilities not discussed in the paper or provided
+ * in the Python prototype.</p>
+ *
+ * <ul><li>The Python prototype only implemented high accuracy for low ranks. This implementation
+ * provides the user with the ability to choose either high rank accuracy or low rank accuracy at
+ * the time of sketch construction.</li>
+ * <li>The Python prototype only implemented a comparison criterion of "INCLUSIVE". This implementation
+ * allows the user to use both the "INCLUSIVE" criterion and the "EXCLUSIVE" criterion.</li>
+ * <li>This implementation provides extensive debug visibility into the operation of the sketch with
+ * two levels of detail output. This is not only useful for debugging, but is a powerful tool to
+ * help users understand how the sketch works.</li>
+ * </ul>
+ */
 template<
   typename T,
   typename Comparator = std::less<T>, // strict weak ordering function (see C++ named requirements: Compare)
@@ -37,8 +80,16 @@ class req_sketch {
 public:
   using value_type = T;
   using comparator = Comparator;
+  using allocator_type = Allocator;
   using Compactor = req_compactor<T, Comparator, Allocator>;
   using AllocCompactor = typename std::allocator_traits<Allocator>::template rebind_alloc<Compactor>;
+  using vector_double = typename quantiles_sorted_view<T, Comparator, Allocator>::vector_double;
+
+  /**
+   * Quantile return type.
+   * This is to return quantiles either by value (for arithmetic types) or by const reference (for all other types)
+   */
+  using quantile_return_type = typename quantiles_sorted_view<T, Comparator, Allocator>::quantile_return_type;
 
   /**
    * Constructor
@@ -46,19 +97,41 @@ public:
    * Value of 12 roughly corresponds to 1% relative error guarantee at 95% confidence.
    * @param hra if true, the default, the high ranks are prioritized for better
    * accuracy. Otherwise the low ranks are prioritized for better accuracy.
-   * @param comparator to use by this instance
-   * @param allocator to use by this instance
+   * @param comparator strict weak ordering function (see C++ named requirements: Compare)
+   * @param allocator used by this sketch to allocate memory
    */
   explicit req_sketch(uint16_t k, bool hra = true, const Comparator& comparator = Comparator(),
       const Allocator& allocator = Allocator());
 
-  ~req_sketch();
+  /**
+   * Copy constructor
+   * @param other sketch to be copied
+   */
   req_sketch(const req_sketch& other);
+
+  /**
+   * Move constructor
+   * @param other sketch to be moved
+   */
   req_sketch(req_sketch&& other) noexcept;
+
+  ~req_sketch();
+
+  /**
+   * Copy assignment
+   * @param other sketch to be copied
+   * @return reference to this sketch
+   */
   req_sketch& operator=(const req_sketch& other);
+
+  /**
+   * Move assignment
+   * @param other sketch to be moved
+   * @return reference to this sketch
+   */
   req_sketch& operator=(req_sketch&& other);
 
-  /*
+  /**
    * Type converting constructor.
    * @param other sketch of a different type
    * @param comparator instance of a Comparator
@@ -177,7 +250,6 @@ public:
    * @return an array of m+1 doubles each of which is an approximation
    * to the fraction of the input stream items (the mass) that fall into one of those intervals.
    */
-  using vector_double = typename quantiles_sorted_view<T, Comparator, Allocator>::vector_double;
   vector_double get_PMF(const T* split_points, uint32_t size, bool inclusive = true) const;
 
   /**
@@ -214,22 +286,8 @@ public:
    *
    * @return approximate quantile associated with the given rank
    */
-  using quantile_return_type = typename quantiles_sorted_view<T, Comparator, Allocator>::quantile_return_type;
   quantile_return_type get_quantile(double rank, bool inclusive = true) const;
 
-  /**
-   * Returns an array of quantiles that correspond to the given array of normalized ranks.
-   * <p>If the sketch is empty this throws std::runtime_error.
-   *
-   * @param ranks given array of normalized ranks.
-   * @param size the number of ranks in the array.
-   *
-   * @return array of quantiles that correspond to the given array of normalized ranks
-   *
-   * Deprecated. Will be removed in the next major version. Use get_quantile() instead.
-   */
-  std::vector<T, Allocator> get_quantiles(const double* ranks, uint32_t size, bool inclusive = true) const;
-
   /**
    * Returns an approximate lower bound of the given normalized rank.
    * @param rank the given rank, a value between 0 and 1.0.
@@ -333,9 +391,26 @@ public:
   string<Allocator> to_string(bool print_levels = false, bool print_items = false) const;
 
   class const_iterator;
+
+  /**
+   * Iterator pointing to the first item in the sketch.
+   * If the sketch is empty, the returned iterator must not be dereferenced or incremented.
+   * @return iterator pointing to the first item in the sketch
+   */
   const_iterator begin() const;
+
+  /**
+   * Iterator pointing to the past-the-end item in the sketch.
+   * The past-the-end item is the hypothetical item that would follow the last item.
+   * It does not point to any item, and must not be dereferenced or incremented.
+   * @return iterator pointing to the past-the-end item in the sketch
+   */
   const_iterator end() const;
 
+  /**
+   * Gets the sorted view of this sketch
+   * @return the sorted view of this sketch
+   */
   quantiles_sorted_view<T, Comparator, Allocator> get_sorted_view() const;
 
 private:
@@ -347,8 +422,8 @@ private:
   uint32_t num_retained_;
   uint64_t n_;
   std::vector<Compactor, AllocCompactor> compactors_;
-  T* min_item_;
-  T* max_item_;
+  optional<T> min_item_;
+  optional<T> max_item_;
   mutable quantiles_sorted_view<T, Comparator, Allocator>* sorted_view_;
 
   void setup_sorted_view() const; // modifies mutable state
@@ -375,9 +450,8 @@ private:
   static bool is_exact_rank(uint16_t k, uint8_t num_levels, double rank, uint64_t n, bool hra);
 
   // for deserialization
-  class item_deleter;
   req_sketch(uint16_t k, bool hra, uint64_t n,
-      std::unique_ptr<T, item_deleter> min_item, std::unique_ptr<T, item_deleter> max_item,
+      optional<T>&& min_item, optional<T>&& max_item,
       std::vector<Compactor, AllocCompactor>&& compactors, const Comparator& comparator);
 
   static void check_preamble_ints(uint8_t preamble_ints, uint8_t num_levels);