datasketches 0.5.2 → 0.5.3

data/vendor/datasketches-cpp/tuple/test/tuple_sketch_test.cpp

@@ -19,6 +19,7 @@
 
 #include <iostream>
 #include <tuple>
+#include <stdexcept>
 
 namespace datasketches {
 
@@ -49,6 +50,49 @@ TEST_CASE("tuple sketch float: builder", "[tuple_sketch]") {
   REQUIRE(sketch.get_theta() == 0.5); // theta = p
 }
 
+TEST_CASE("tuple sketch: min lg_k", "[tuple_sketch]") {
+  // Tuple sketches reuse theta's builder and hash table, so the same nominal floor
+  // (MIN_LG_K = 4, matching Java's ThetaUtil.MIN_LG_NOM_LONGS) and cache floor (MIN_LG_ARR = 5)
+  // apply. lg_k = 4 (nominal 16) is the smallest allowed nominal size; below it must throw.
+  REQUIRE(theta_constants::MIN_LG_K == 4);
+  REQUIRE_THROWS_AS(update_tuple_sketch<float>::builder().set_lg_k(theta_constants::MIN_LG_K - 1),
+      std::invalid_argument);
+  auto min_sketch = update_tuple_sketch<float>::builder().set_lg_k(theta_constants::MIN_LG_K).build();
+  REQUIRE(min_sketch.get_lg_k() == theta_constants::MIN_LG_K);
+
+  // update well past the nominal size to force estimation mode and exercise the rebuild path,
+  // tracking the peak number of retained entries seen between rebuilds.
+  const int n = 10000;
+  uint32_t max_retained = 0;
+  for (int i = 0; i < n; ++i) {
+    min_sketch.update(i, 1.0f);
+    if (min_sketch.get_num_retained() > max_retained) max_retained = min_sketch.get_num_retained();
+  }
+  REQUIRE(min_sketch.is_estimation_mode());
+  REQUIRE(min_sketch.get_theta() < 1.0);
+
+  // The internal hash table is floored at MIN_LG_ARR (5, i.e. 32 slots), one lg above the
+  // nominal size. This is exactly what MIN_LG_ARR guarantees: between rebuilds the sketch holds
+  // more than the nominal 2^MIN_LG_K (16) entries, but never more than the 2^MIN_LG_ARR (32)
+  // slots of the table. Were the table sized to the nominal 16, it could not retain more than 16.
+  REQUIRE(max_retained > (1 << theta_constants::MIN_LG_K));
+  REQUIRE(max_retained <= (1 << theta_constants::MIN_LG_ARR));
+
+  // the true count is bracketed by the 2-standard-deviation confidence bounds
+  REQUIRE(min_sketch.get_lower_bound(2) <= n);
+  REQUIRE(min_sketch.get_upper_bound(2) >= n);
+
+  // trimming reduces the sketch to exactly the nominal number of entries (2^4 = 16)
+  min_sketch.trim();
+  REQUIRE(min_sketch.get_num_retained() == (1 << theta_constants::MIN_LG_K));
+
+  // a compacted min-size sketch round trips through serialization
+  auto bytes = min_sketch.compact().serialize();
+  auto deserialized = compact_tuple_sketch<float>::deserialize(bytes.data(), bytes.size());
+  REQUIRE(deserialized.get_num_retained() == min_sketch.get_num_retained());
+  REQUIRE(deserialized.get_estimate() == min_sketch.get_estimate());
+}
+
 TEST_CASE("tuple sketch float: empty", "[tuple_sketch]") {
   auto update_sketch = update_tuple_sketch<float>::builder().build();
   std::cout << "sizeof(update_tuple_sketch<float>)=" << sizeof(update_sketch) << std::endl;
@@ -371,4 +415,19 @@ TEST_CASE("filter", "[tuple_sketch]") {
   }
 }
 
+TEST_CASE("tuple sketch: deserialize bounds-checks each entry key", "[tuple_sketch]") {
+  // A compact sketch serialized with a narrower summary (float, 4 bytes) and then
+  // deserialized as a wider summary (double, 8 bytes). The per-entry stride the reader
+  // assumes (8-byte key + 8-byte summary) is larger than the entries actually occupy
+  // (8-byte key + 4-byte summary), so the read cursor advances past the end of the buffer.
+  // num_entries is read from the preamble and is unaffected, so the entry loop still runs
+  // the full count and the per-entry key read walks off the end. This must throw rather
+  // than read out of bounds (a heap-buffer-overflow under AddressSanitizer).
+  auto update_sketch = update_tuple_sketch<float>::builder().build();
+  for (int i = 0; i < 100; ++i) update_sketch.update(i, 1.0f);
+  auto bytes = update_sketch.compact().serialize();
+  REQUIRE_THROWS_AS(compact_tuple_sketch<double>::deserialize(bytes.data(), bytes.size()),
+                    std::out_of_range);
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/version.cfg.in

@@ -1 +1 @@
-5.2.0
+5.3.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datasketches
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Andrew Kane
@@ -50,6 +50,9 @@ files:
 - vendor/datasketches-cpp/LICENSE
 - vendor/datasketches-cpp/NOTICE
 - vendor/datasketches-cpp/README.md
+- vendor/datasketches-cpp/benchmarks/CMakeLists.txt
+- vendor/datasketches-cpp/benchmarks/benchmark_count_min_sketch.cpp
+- vendor/datasketches-cpp/benchmarks/benchmark_count_min_sketch_serialization.cpp
 - vendor/datasketches-cpp/cmake/DataSketchesConfig.cmake.in
 - vendor/datasketches-cpp/common/CMakeLists.txt
 - vendor/datasketches-cpp/common/include/MurmurHash3.h
@@ -60,6 +63,7 @@ files:
 - vendor/datasketches-cpp/common/include/conditional_back_inserter.hpp
 - vendor/datasketches-cpp/common/include/conditional_forward.hpp
 - vendor/datasketches-cpp/common/include/count_zeros.hpp
+- vendor/datasketches-cpp/common/include/fdlibm_log.hpp
 - vendor/datasketches-cpp/common/include/inv_pow2_table.hpp
 - vendor/datasketches-cpp/common/include/kolmogorov_smirnov.hpp
 - vendor/datasketches-cpp/common/include/kolmogorov_smirnov_impl.hpp
@@ -71,7 +75,9 @@ files:
 - vendor/datasketches-cpp/common/include/version.hpp.in
 - vendor/datasketches-cpp/common/include/xxhash64.h
 - vendor/datasketches-cpp/common/test/CMakeLists.txt
+- vendor/datasketches-cpp/common/test/binomial_bounds_test.cpp
 - vendor/datasketches-cpp/common/test/catch_runner.cpp
+- vendor/datasketches-cpp/common/test/deserialize_hardening_test.cpp
 - vendor/datasketches-cpp/common/test/integration_test.cpp
 - vendor/datasketches-cpp/common/test/optional_test.cpp
 - vendor/datasketches-cpp/common/test/quantiles_sorted_view_test.cpp
@@ -172,6 +178,8 @@ files:
 - vendor/datasketches-cpp/hll/test/CouponListTest.cpp
 - vendor/datasketches-cpp/hll/test/CrossCountingTest.cpp
 - vendor/datasketches-cpp/hll/test/HllArrayTest.cpp
+- vendor/datasketches-cpp/hll/test/HllFullSizeTest.cpp
+- vendor/datasketches-cpp/hll/test/HllKxqRebuildTest.cpp
 - vendor/datasketches-cpp/hll/test/HllSketchTest.cpp
 - vendor/datasketches-cpp/hll/test/HllUnionTest.cpp
 - vendor/datasketches-cpp/hll/test/IsomorphicTest.cpp
@@ -246,6 +254,7 @@ files:
 - vendor/datasketches-cpp/tdigest/test/CMakeLists.txt
 - vendor/datasketches-cpp/tdigest/test/tdigest_custom_allocator_test.cpp
 - vendor/datasketches-cpp/tdigest/test/tdigest_deserialize_from_java_test.cpp
+- vendor/datasketches-cpp/tdigest/test/tdigest_iterator_test.cpp
 - vendor/datasketches-cpp/tdigest/test/tdigest_ref_k100_n10000_double.sk
 - vendor/datasketches-cpp/tdigest/test/tdigest_ref_k100_n10000_float.sk
 - vendor/datasketches-cpp/tdigest/test/tdigest_serialize_for_java.cpp
@@ -291,6 +300,7 @@ files:
 - vendor/datasketches-cpp/theta/test/theta_sketch_serialize_for_java.cpp
 - vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp
 - vendor/datasketches-cpp/theta/test/theta_union_test.cpp
+- vendor/datasketches-cpp/tools/rat-check.sh
 - vendor/datasketches-cpp/tuple/CMakeLists.txt
 - vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch.hpp
 - vendor/datasketches-cpp/tuple/include/array_tuple_a_not_b.hpp
@@ -342,7 +352,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Sketch data structures for Ruby
 test_files: []