datasketches 0.3.2 → 0.4.1

data/vendor/datasketches-cpp/hll/test/hll_sketch_deserialize_from_java_test.cpp

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <hll.hpp>
+
+namespace datasketches {
+
+// assume the binary sketches for this test have been generated by datasketches-java code
+// in the subdirectory called "java" in the root directory of this project
+static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../java/";
+
+TEST_CASE("hll4 sketch", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "hll4_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = hll_sketch::deserialize(is);
+    REQUIRE(sketch.get_lg_config_k() == 12);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.02));
+  }
+}
+
+TEST_CASE("hll6 sketch", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "hll6_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = hll_sketch::deserialize(is);
+    REQUIRE(sketch.get_lg_config_k() == 12);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.02));
+  }
+}
+
+TEST_CASE("hll8 sketch", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "hll8_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = hll_sketch::deserialize(is);
+    REQUIRE(sketch.get_lg_config_k() == 12);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.02));
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/hll/test/hll_sketch_serialize_for_java.cpp

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <hll.hpp>
+
+namespace datasketches {
+
+TEST_CASE("hll sketch generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    hll_sketch hll4(12, HLL_4);
+    hll_sketch hll6(12, HLL_6);
+    hll_sketch hll8(12, HLL_8);
+    for (unsigned i = 0; i < n; ++i) {
+      hll4.update(i);
+      hll6.update(i);
+      hll8.update(i);
+    }
+    {
+      std::ofstream os("hll4_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+      hll4.serialize_compact(os);
+    }
+    {
+      std::ofstream os("hll6_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+      hll6.serialize_compact(os);
+    }
+    {
+      std::ofstream os("hll8_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+      hll8.serialize_compact(os);
+    }
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/kll/include/kll_helper_impl.hpp

@@ -99,7 +99,7 @@ void kll_helper::randomly_halve_down(T* buf, uint32_t start, uint32_t length) {
 #ifdef KLL_VALIDATION
   const uint32_t offset = deterministic_offset();
 #else
-  const uint32_t offset = random_bit();
+  const uint32_t offset = random_utils::random_bit();
 #endif
   uint32_t j = start + offset;
   for (uint32_t i = start; i < (start + half_length); i++) {
@@ -115,7 +115,7 @@ void kll_helper::randomly_halve_up(T* buf, uint32_t start, uint32_t length) {
 #ifdef KLL_VALIDATION
   const uint32_t offset = deterministic_offset();
 #else
-  const uint32_t offset = random_bit();
+  const uint32_t offset = random_utils::random_bit();
 #endif
   uint32_t j = (start + length) - 1 - offset;
   for (uint32_t i = (start + length) - 1; i >= (start + half_length); i--) {
@@ -230,7 +230,8 @@ kll_helper::compress_result kll_helper::general_compress(uint16_t k, uint8_t m,
       // move level over as is
       // make sure we are not moving data upwards
       if (raw_beg < out_levels[current_level]) throw std::logic_error("wrong move");
-      std::move(items + raw_beg, items + raw_lim, items + out_levels[current_level]);
+      if (raw_beg != out_levels[current_level])
+        std::move(items + raw_beg, items + raw_lim, items + out_levels[current_level]);
       out_levels[current_level + 1] = out_levels[current_level] + raw_pop;
     } else {
       // The sketch is too full AND this level is too full, so we compact it
@@ -243,7 +244,8 @@ kll_helper::compress_result kll_helper::general_compress(uint16_t k, uint8_t m,
       const auto half_adj_pop = adj_pop / 2;
 
       if (odd_pop) { // move one guy over
-        items[out_levels[current_level]] = std::move(items[raw_beg]);
+        if (out_levels[current_level] != raw_beg)
+          items[out_levels[current_level]] = std::move(items[raw_beg]);
         out_levels[current_level + 1] = out_levels[current_level] + 1;
       } else { // even number of items
         out_levels[current_level + 1] = out_levels[current_level];

data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp

@@ -26,10 +26,22 @@
 #include "common_defs.hpp"
 #include "serde.hpp"
 #include "quantiles_sorted_view.hpp"
+#include "optional.hpp"
 
 namespace datasketches {
 
-/*
+/// KLL sketch constants
+namespace kll_constants {
+  /// default value of parameter K
+  const uint16_t DEFAULT_K = 200;
+  const uint8_t DEFAULT_M = 8;
+  /// min value of parameter K
+  const uint16_t MIN_K = DEFAULT_M;
+  /// max value of parameter K
+  const uint16_t MAX_K = (1 << 16) - 1;
+}
+
+/**
  * Implementation of a very compact quantiles sketch with lazy compaction scheme
  * and nearly optimal accuracy per retained item.
  * See <a href="https://arxiv.org/abs/1603.05346v2">Optimal Quantile Approximation in Streams</a>.
@@ -146,10 +158,6 @@ namespace datasketches {
  * author Lee Rhodes
  */
 
-namespace kll_constants {
-  const uint16_t DEFAULT_K = 200;
-}
-
 template <
   typename T,
   typename C = std::less<T>, // strict weak ordering function (see C++ named requirements: Compare)
@@ -159,17 +167,51 @@ class kll_sketch {
   public:
     using value_type = T;
     using comparator = C;
+    using allocator_type = A;
     using vector_u32 = std::vector<uint32_t, typename std::allocator_traits<A>::template rebind_alloc<uint32_t>>;
+    using vector_double = typename quantiles_sorted_view<T, C, A>::vector_double;
 
-    static const uint8_t DEFAULT_M = 8;
-    static const uint16_t MIN_K = DEFAULT_M;
-    static const uint16_t MAX_K = (1 << 16) - 1;
+    /**
+     * Quantile return type.
+     * This is to return quantiles either by value (for arithmetic types) or by const reference (for all other types)
+     */
+    using quantile_return_type = typename quantiles_sorted_view<T, C, A>::quantile_return_type;
 
+    /**
+     * Constructor
+     * @param k affects the size of the sketch and its estimation error
+     * @param comparator strict weak ordering function (see C++ named requirements: Compare)
+     * @param allocator used by this sketch to allocate memory
+     */
     explicit kll_sketch(uint16_t k = kll_constants::DEFAULT_K, const C& comparator = C(), const A& allocator = A());
+
+    /**
+     * Copy constructor
+     * @param other sketch to be copied
+     */
     kll_sketch(const kll_sketch& other);
+
+    /**
+     * Move constructor
+     * @param other sketch to be moved
+     */
     kll_sketch(kll_sketch&& other) noexcept;
+
+
     ~kll_sketch();
+
+    /**
+     * Copy assignment
+     * @param other sketch to be copied
+     * @return reference to this sketch
+     */
     kll_sketch& operator=(const kll_sketch& other);
+
+    /**
+     * Move assignment
+     * @param other sketch to be moved
+     * @return reference to this sketch
+     */
     kll_sketch& operator=(kll_sketch&& other);
 
     /*
@@ -262,44 +304,8 @@ class kll_sketch {
      *
      * @return approximate quantile associated with the given rank
      */
-    using quantile_return_type = typename quantiles_sorted_view<T, C, A>::quantile_return_type;
     quantile_return_type get_quantile(double rank, bool inclusive = true) const;
 
-    /**
-     * This returns an array that could have been generated by using get_quantile() for each
-     * rank separately.
-     *
-     * <p>If the sketch is empty this throws std::runtime_error.
-     *
-     * @param ranks given array of ranks in the hypothetical sorted stream.
-     * These ranks must be in the interval [0.0, 1.0].
-     * @param size the number of ranks in the array
-     * @param inclusive if true, the given ranks are considered inclusive (include weights of items)
-     *
-     * @return array of approximate quantiles corresponding to the given ranks in the same order.
-     *
-     * Deprecated. Will be removed in the next major version. Use get_quantile() instead.
-     */
-    std::vector<T, A> get_quantiles(const double* ranks, uint32_t size, bool inclusive = true) const;
-
-    /**
-     * This is a multiple-query version of get_quantile() that allows the caller to
-     * specify the number of evenly-spaced ranks.
-     *
-     * <p>If the sketch is empty this throws std::runtime_error.
-     *
-     * @param num an integer that specifies the number of evenly-spaced ranks.
-     * This must be an integer greater than 0. A value of 1 will return the quantile of rank 0.
-     * A value of 2 will return quantiles of ranks 0 and 1. A value of 3 will return quantiles of ranks 0,
-     * 0.5 (median) and 1, etc.
-     * @param inclusive if true, the ranks are considered inclusive (include weights of items)
-     *
-     * @return array of approximate quantiles corresponding to the given number of evenly-spaced ranks.
-     *
-     * Deprecated. Will be removed in the next major version. Use get_quantile() instead.
-     */
-    std::vector<T, A> get_quantiles(uint32_t num, bool inclusive = true) const;
-
     /**
      * Returns an approximation to the normalized rank of the given item from 0 to 1, inclusive.
      *
@@ -339,7 +345,6 @@ class kll_sketch {
      * @return an array of m+1 doubles each of which is an approximation
      * to the fraction of the input stream items (the mass) that fall into one of those intervals.
      */
-    using vector_double = typename quantiles_sorted_view<T, C, A>::vector_double;
     vector_double get_PMF(const T* split_points, uint32_t size, bool inclusive = true) const;
 
     /**
@@ -489,9 +494,26 @@ class kll_sketch {
     string<A> to_string(bool print_levels = false, bool print_items = false) const;
 
     class const_iterator;
+
+    /**
+     * Iterator pointing to the first item in the sketch.
+     * If the sketch is empty, the returned iterator must not be dereferenced or incremented.
+     * @return iterator pointing to the first item in the sketch
+     */
     const_iterator begin() const;
+
+    /**
+     * Iterator pointing to the past-the-end item in the sketch.
+     * The past-the-end item is the hypothetical item that would follow the last item.
+     * It does not point to any item, and must not be dereferenced or incremented.
+     * @return iterator pointing to the past-the-end item in the sketch
+     */
     const_iterator end() const;
 
+    /**
+     * Gets the sorted view of this sketch
+     * @return the sorted view of this sketch
+     */
     quantiles_sorted_view<T, C, A> get_sorted_view() const;
 
   private:
@@ -529,16 +551,15 @@ class kll_sketch {
     vector_u32 levels_;
     T* items_;
     uint32_t items_size_;
-    T* min_item_;
-    T* max_item_;
+    optional<T> min_item_;
+    optional<T> max_item_;
     mutable quantiles_sorted_view<T, C, A>* sorted_view_;
 
     // for deserialization
-    class item_deleter;
     class items_deleter;
     kll_sketch(uint16_t k, uint16_t min_k, uint64_t n, uint8_t num_levels, vector_u32&& levels,
-        std::unique_ptr<T, items_deleter> items, uint32_t items_size, std::unique_ptr<T, item_deleter> min_item,
-        std::unique_ptr<T, item_deleter> max_item, bool is_level_zero_sorted, const C& comparator);
+        std::unique_ptr<T, items_deleter> items, uint32_t items_size, optional<T>&& min_item,
+        optional<T>&& max_item, bool is_level_zero_sorted, const C& comparator);
 
     // common update code
     inline void update_min_max(const T& item);