datasketches 0.5.1 → 0.5.3

data/vendor/datasketches-cpp/common/test/binomial_bounds_test.cpp

@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+
+#include "binomial_bounds.hpp"
+
+namespace datasketches {
+
+TEST_CASE("binomial_bounds: get_lower_bound", "[common]") {
+
+  SECTION("num_samples == 0") {
+    double result = binomial_bounds::get_lower_bound(0, 0.5, 1);
+    REQUIRE(result == 0.0);
+  }
+
+  SECTION("theta == 1.0") {
+    double result = binomial_bounds::get_lower_bound(100, 1.0, 1);
+    REQUIRE(result == 100.0);
+  }
+
+  SECTION("num_samples == 1") {
+    double result = binomial_bounds::get_lower_bound(1, 0.5, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples == 1, stddev=2") {
+    double result = binomial_bounds::get_lower_bound(1, 0.5, 2);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples == 1, stddev=3") {
+    double result = binomial_bounds::get_lower_bound(1, 0.5, 3);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples > 120") {
+    double result = binomial_bounds::get_lower_bound(121, 0.5, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples > 120, stddev=2") {
+    double result = binomial_bounds::get_lower_bound(200, 0.5, 2);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples > 120, stddev=3") {
+    double result = binomial_bounds::get_lower_bound(500, 0.5, 3);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("2 <= num_samples <= 120 AND theta > (1-1e-5)") {
+    double result = binomial_bounds::get_lower_bound(50, 1.0 - 1e-6, 1);
+    REQUIRE(std::abs(result - 50.0) < 50.0 * 0.01);
+  }
+
+  SECTION("2 <= num_samples <= 120 AND theta > (1-1e-5), stddev=2") {
+    double result = binomial_bounds::get_lower_bound(50, 1.0 - 1e-6, 2);
+    REQUIRE(std::abs(result - 50.0) < 50.0 * 0.01);
+  }
+
+  SECTION("2 <= num_samples <= 120 AND theta > (1-1e-5), stddev=3") {
+    double result = binomial_bounds::get_lower_bound(50, 1.0 - 1e-6, 3);
+    REQUIRE(std::abs(result - 50.0) < 50.0 * 0.01);
+  }
+
+  SECTION("2 <= num_samples <= 120 AND theta < num_samples/360") {
+    double result = binomial_bounds::get_lower_bound(100, 0.001, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("2 <= num_samples <= 120 AND theta < num_samples/360, stddev=2") {
+    double result = binomial_bounds::get_lower_bound(100, 0.001, 2);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("2 <= num_samples <= 120 AND theta < num_samples/360, stddev=3") {
+    double result = binomial_bounds::get_lower_bound(100, 0.001, 3);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("2 <= num_samples <= 120 AND middle range theta (exact calculation)") {
+    double result = binomial_bounds::get_lower_bound(10, 0.5, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("2 <= num_samples <= 120 AND middle range theta, stddev=2") {
+    double result = binomial_bounds::get_lower_bound(10, 0.5, 2);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("2 <= num_samples <= 120 AND middle range theta, stddev=3") {
+    double result = binomial_bounds::get_lower_bound(10, 0.5, 3);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("theta=0") {
+    REQUIRE_THROWS_AS(binomial_bounds::get_lower_bound(10, 0.0, 1), std::invalid_argument);
+  }
+
+  SECTION("theta very close to 0") {
+    double result = binomial_bounds::get_lower_bound(10, 1e-10, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples=2 boundary") {
+    double result = binomial_bounds::get_lower_bound(2, 0.5, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples=120 boundary") {
+    double result = binomial_bounds::get_lower_bound(120, 0.5, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("estimate clamping case") {
+    double result = binomial_bounds::get_lower_bound(10, 0.9, 1);
+    double estimate = 10.0 / 0.9;
+    REQUIRE(result <= estimate);
+  }
+
+  SECTION("invalid theta < 0") {
+    REQUIRE_THROWS_AS(binomial_bounds::get_lower_bound(100, -0.1, 1), std::invalid_argument);
+  }
+
+  SECTION("invalid theta > 1") {
+    REQUIRE_THROWS_AS(binomial_bounds::get_lower_bound(100, 1.1, 1), std::invalid_argument);
+  }
+
+  SECTION("invalid stddev = 0") {
+    REQUIRE_THROWS_AS(binomial_bounds::get_lower_bound(100, 0.5, 0), std::invalid_argument);
+  }
+
+  SECTION("invalid stddev = 4") {
+    REQUIRE_THROWS_AS(binomial_bounds::get_lower_bound(100, 0.5, 4), std::invalid_argument);
+  }
+}
+
+TEST_CASE("binomial_bounds: get_upper_bound", "[common]") {
+
+  SECTION("theta == 1.0") {
+    double result = binomial_bounds::get_upper_bound(100, 1.0, 1);
+    REQUIRE(result == 100.0);
+  }
+
+  SECTION("num_samples == 0") {
+    double result = binomial_bounds::get_upper_bound(0, 0.5, 1);
+    REQUIRE(result > 0.0);
+  }
+
+  SECTION("num_samples == 0, stddev=2") {
+    double result = binomial_bounds::get_upper_bound(0, 0.5, 2);
+    REQUIRE(result > 0.0);
+  }
+
+  SECTION("num_samples == 0, stddev=3") {
+    double result = binomial_bounds::get_upper_bound(0, 0.5, 3);
+    REQUIRE(result > 0.0);
+  }
+
+  SECTION("num_samples > 120") {
+    double result = binomial_bounds::get_upper_bound(121, 0.5, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples > 120, stddev=2") {
+    double result = binomial_bounds::get_upper_bound(200, 0.5, 2);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples > 120, stddev=3") {
+    double result = binomial_bounds::get_upper_bound(500, 0.5, 3);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("1 <= num_samples <= 120 AND theta > (1-1e-5)") {
+    double result = binomial_bounds::get_upper_bound(50, 1.0 - 1e-6, 1);
+    REQUIRE(result == 51.0);
+  }
+
+  SECTION("1 <= num_samples <= 120 AND theta > (1-1e-5), stddev=2") {
+    double result = binomial_bounds::get_upper_bound(50, 1.0 - 1e-6, 2);
+    REQUIRE(result == 51.0);
+  }
+
+  SECTION("1 <= num_samples <= 120 AND theta > (1-1e-5), stddev=3") {
+    double result = binomial_bounds::get_upper_bound(50, 1.0 - 1e-6, 3);
+    REQUIRE(result == 51.0);
+  }
+
+  SECTION("1 <= num_samples <= 120 AND theta < num_samples/360") {
+    double result = binomial_bounds::get_upper_bound(100, 0.001, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("1 <= num_samples <= 120 AND theta < num_samples/360, stddev=2") {
+    double result = binomial_bounds::get_upper_bound(100, 0.001, 2);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("1 <= num_samples <= 120 AND theta < num_samples/360, stddev=3") {
+    double result = binomial_bounds::get_upper_bound(100, 0.001, 3);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("1 <= num_samples <= 120 AND middle range theta (exact calculation)") {
+    double result = binomial_bounds::get_upper_bound(10, 0.5, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("1 <= num_samples <= 120 AND middle range theta, stddev=2") {
+    double result = binomial_bounds::get_upper_bound(10, 0.5, 2);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("1 <= num_samples <= 120 AND middle range theta, stddev=3") {
+    double result = binomial_bounds::get_upper_bound(10, 0.5, 3);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("theta=0") {
+    REQUIRE_THROWS_AS(binomial_bounds::get_upper_bound(10, 0.0, 1), std::invalid_argument);
+  }
+
+  SECTION("theta very close to 0") {
+    double result = binomial_bounds::get_upper_bound(10, 1e-10, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples=1 boundary") {
+    double result = binomial_bounds::get_upper_bound(1, 0.5, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("num_samples=120 boundary") {
+    double result = binomial_bounds::get_upper_bound(120, 0.5, 1);
+    REQUIRE(result >= 0.0);
+  }
+
+  SECTION("estimate clamping case") {
+    double result = binomial_bounds::get_upper_bound(10, 0.9, 1);
+    double estimate = 10.0 / 0.9;
+    REQUIRE(result >= estimate);
+  }
+
+  SECTION("invalid theta < 0") {
+    REQUIRE_THROWS_AS(binomial_bounds::get_upper_bound(100, -0.1, 1), std::invalid_argument);
+  }
+
+  SECTION("invalid theta > 1") {
+    REQUIRE_THROWS_AS(binomial_bounds::get_upper_bound(100, 1.1, 1), std::invalid_argument);
+  }
+
+  SECTION("invalid stddev = 0") {
+    REQUIRE_THROWS_AS(binomial_bounds::get_upper_bound(100, 0.5, 0), std::invalid_argument);
+  }
+
+  SECTION("invalid stddev = 4") {
+    REQUIRE_THROWS_AS(binomial_bounds::get_upper_bound(100, 0.5, 4), std::invalid_argument);
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/common/test/deserialize_hardening_test.cpp

@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <sstream>
+#include <vector>
+
+// Include all affected sketch types
+#include <quantiles_sketch.hpp>
+#include <kll_sketch.hpp>
+#include <req_sketch.hpp>
+
+namespace datasketches {
+
+/**
+ * Test for fix of issue #477:
+ * BUG: SIGABRT in deserialize(): dereferencing empty std::optional (libc++ verbose_abort)
+ * 
+ * These tests exercise the actual deserialization code path that contained the bug.
+ * With buggy code (&*tmp on empty optional) and hardening enabled, these will SIGABRT.
+ * With fixed code (aligned_storage), these pass normally.
+ * 
+ * IMPORTANT: These tests actually call deserialize() on multi-item sketches, which
+ * exercises the buggy code path where min/max are deserialized.
+ */
+
+TEST_CASE("quantiles_sketch: deserialize multi-item sketch", "[deserialize_hardening]") {
+  // Create sketch with multiple items (so min/max are stored in serialization)
+  quantiles_sketch<double> sketch(128);
+  for (int i = 0; i < 1000; i++) {
+    sketch.update(static_cast<double>(i));
+  }
+  
+  // Serialize
+  auto bytes = sketch.serialize();
+  
+  // Deserialize - WITH BUGGY CODE AND HARDENING, THIS WILL SIGABRT HERE
+  // The bug is: sd.deserialize(is, &*tmp, 1) where tmp is empty optional
+  auto sketch2 = quantiles_sketch<double>::deserialize(bytes.data(), bytes.size());
+  
+  // Verify deserialization worked correctly
+  REQUIRE(sketch2.get_n() == sketch.get_n());
+  REQUIRE(sketch2.get_min_item() == sketch.get_min_item());
+  REQUIRE(sketch2.get_max_item() == sketch.get_max_item());
+  REQUIRE(sketch2.get_quantile(0.5) == sketch.get_quantile(0.5));
+}
+
+TEST_CASE("quantiles_sketch: deserialize from stream", "[deserialize_hardening]") {
+  quantiles_sketch<float> sketch(256);
+  for (int i = 0; i < 2000; i++) {
+    sketch.update(static_cast<float>(i) * 0.5f);
+  }
+  
+  // Serialize to stream
+  std::stringstream ss;
+  sketch.serialize(ss);
+  
+  // Deserialize from stream - exercises the buggy code path
+  auto sketch2 = quantiles_sketch<float>::deserialize(ss);
+  
+  REQUIRE(sketch2.get_n() == sketch.get_n());
+  REQUIRE(sketch2.get_min_item() == sketch.get_min_item());
+  REQUIRE(sketch2.get_max_item() == sketch.get_max_item());
+}
+
+TEST_CASE("kll_sketch: deserialize multi-item sketch", "[deserialize_hardening]") {
+  kll_sketch<float> sketch(200);
+  for (int i = 0; i < 1500; i++) {
+    sketch.update(static_cast<float>(i));
+  }
+  
+  auto bytes = sketch.serialize();
+  
+  // Deserialize - exercises buggy &*tmp code path
+  auto sketch2 = kll_sketch<float>::deserialize(bytes.data(), bytes.size());
+  
+  REQUIRE(sketch2.get_n() == sketch.get_n());
+  REQUIRE(sketch2.get_min_item() == sketch.get_min_item());
+  REQUIRE(sketch2.get_max_item() == sketch.get_max_item());
+}
+
+TEST_CASE("kll_sketch: deserialize from stream", "[deserialize_hardening]") {
+  kll_sketch<int> sketch(400);
+  for (int i = 0; i < 3000; i++) {
+    sketch.update(i);
+  }
+  
+  std::stringstream ss;
+  sketch.serialize(ss);
+  
+  // Deserialize from stream
+  auto sketch2 = kll_sketch<int>::deserialize(ss);
+  
+  REQUIRE(sketch2.get_n() == sketch.get_n());
+  REQUIRE(sketch2.get_min_item() == sketch.get_min_item());
+  REQUIRE(sketch2.get_max_item() == sketch.get_max_item());
+}
+
+TEST_CASE("req_sketch: deserialize multi-level sketch", "[deserialize_hardening]") {
+  // REQ sketch only has the bug when num_levels > 1
+  // We need to add enough items to trigger multiple levels
+  req_sketch<float> sketch(12);
+  for (int i = 0; i < 10000; i++) {
+    sketch.update(static_cast<float>(i));
+  }
+  
+  auto bytes = sketch.serialize();
+  
+  // Deserialize - exercises buggy code path when num_levels > 1
+  auto sketch2 = req_sketch<float>::deserialize(bytes.data(), bytes.size());
+  
+  REQUIRE(sketch2.get_n() == sketch.get_n());
+  REQUIRE(sketch2.get_min_item() == sketch.get_min_item());
+  REQUIRE(sketch2.get_max_item() == sketch.get_max_item());
+}
+
+TEST_CASE("req_sketch: deserialize from stream", "[deserialize_hardening]") {
+  req_sketch<double> sketch(20);
+  for (int i = 0; i < 15000; i++) {
+    sketch.update(static_cast<double>(i) * 0.1);
+  }
+  
+  std::stringstream ss;
+  sketch.serialize(ss);
+  
+  // Deserialize from stream
+  auto sketch2 = req_sketch<double>::deserialize(ss);
+  
+  REQUIRE(sketch2.get_n() == sketch.get_n());
+  REQUIRE(sketch2.get_min_item() == sketch.get_min_item());
+  REQUIRE(sketch2.get_max_item() == sketch.get_max_item());
+}
+
+TEST_CASE("multiple sketch types: stress test", "[deserialize_hardening]") {
+  SECTION("quantiles with various sizes") {
+    for (int k : {64, 128, 256}) {
+      quantiles_sketch<int> sketch(k);
+      for (int i = 0; i < 5000; i++) {
+        sketch.update(i);
+      }
+      auto bytes = sketch.serialize();
+      auto sketch2 = quantiles_sketch<int>::deserialize(bytes.data(), bytes.size());
+      REQUIRE(sketch2.get_n() == sketch.get_n());
+    }
+  }
+  
+  SECTION("kll with various sizes") {
+    for (int k : {100, 200, 400}) {
+      kll_sketch<double> sketch(k);
+      for (int i = 0; i < 4000; i++) {
+        sketch.update(static_cast<double>(i) / 10.0);
+      }
+      auto bytes = sketch.serialize();
+      auto sketch2 = kll_sketch<double>::deserialize(bytes.data(), bytes.size());
+      REQUIRE(sketch2.get_n() == sketch.get_n());
+    }
+  }
+  
+  SECTION("req with various sizes") {
+    for (int k : {12, 20}) {
+      req_sketch<float> sketch(k);
+      for (int i = 0; i < 8000; i++) {
+        sketch.update(static_cast<float>(i));
+      }
+      auto bytes = sketch.serialize();
+      auto sketch2 = req_sketch<float>::deserialize(bytes.data(), bytes.size());
+      REQUIRE(sketch2.get_n() == sketch.get_n());
+    }
+  }
+}
+
+} // namespace datasketches

data/vendor/datasketches-cpp/count/include/count_min.hpp

@@ -21,6 +21,8 @@
 #define COUNT_MIN_HPP_
 
 #include <iterator>
+#include <type_traits>
+#include <vector>
 #include "common_defs.hpp"
 
 namespace datasketches {
@@ -36,6 +38,7 @@ template <typename W,
           typename Allocator = std::allocator<W>>
 class count_min_sketch{
   static_assert(std::is_arithmetic<W>::value, "Arithmetic type expected");
+  static_assert(!std::is_same<W, bool>::value, "Boolean weight type is not supported");
 public:
   using allocator_type = Allocator;
   using const_iterator = typename std::vector<W, Allocator>::const_iterator;
@@ -326,6 +329,15 @@ public:
    */
   void serialize(std::ostream& os) const;
 
+  /**
+   * This method serializes the sketch by passing binary fragments to a callback.
+   * The callback must be callable as write_bytes(const void* data, size_t size).
+   * The data pointer is valid only until the callback returns.
+   * @return size in bytes written to the callback
+   */
+  template<typename Sink>
+  size_t serialize_to(Sink&& sink) const;
+
   // This is a convenience alias for users
   // The type returned by the following serialize method
   using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<Allocator>::template rebind_alloc<uint8_t>>;
@@ -389,12 +401,13 @@ private:
   static void check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id, uint8_t flags_byte);
 
   /*
-   * Obtain the hash values when inserting an item into the sketch.
-   * @param item pointer to the data item to be inserted into the sketch.
+   * Compute the hash locations for an input item
+   * @param item pointer to the data item to be inserted into or queried from the sketch.
    * @param size of the data in bytes
-   * @return vector of uint64_t which each represent the index to which `value' must update in the sketch
+   * @param callback function to invoke for each sketch array location
    */
-  std::vector<uint64_t> get_hashes(const void* item, size_t size) const;
+  template<typename F>
+  void foreach_hash_location(const void* item, size_t size, F callback) const;
 
 };