datasketches 0.5.3 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/lib/datasketches/version.rb +1 -1
  4. data/vendor/datasketches-cpp/LICENSE +0 -34
  5. data/vendor/datasketches-cpp/NOTICE +2 -2
  6. data/vendor/datasketches-cpp/README.md +1 -1
  7. data/vendor/datasketches-cpp/common/include/memory_operations.hpp +2 -2
  8. data/vendor/datasketches-cpp/count/include/count_min.hpp +1 -1
  9. data/vendor/datasketches-cpp/count/include/count_min_impl.hpp +2 -1
  10. data/vendor/datasketches-cpp/count/test/count_min_allocation_test.cpp +17 -0
  11. data/vendor/datasketches-cpp/density/include/density_sketch.hpp +3 -2
  12. data/vendor/datasketches-cpp/density/test/density_sketch_test.cpp +30 -0
  13. data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp +18 -9
  14. data/vendor/datasketches-cpp/hll/include/hll.hpp +1 -1
  15. data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +73 -0
  16. data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +3 -0
  17. data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +35 -0
  18. data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +70 -0
  19. data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch_impl.hpp +1 -0
  20. data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_test.cpp +13 -0
  21. data/vendor/datasketches-cpp/req/include/req_common.hpp +2 -0
  22. data/vendor/datasketches-cpp/req/include/req_compactor.hpp +3 -2
  23. data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +31 -3
  24. data/vendor/datasketches-cpp/req/include/req_sketch.hpp +5 -1
  25. data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +31 -3
  26. data/vendor/datasketches-cpp/req/test/req_sketch_test.cpp +176 -0
  27. data/vendor/datasketches-cpp/sampling/include/ebpps_sketch_impl.hpp +1 -1
  28. data/vendor/datasketches-cpp/sampling/test/ebpps_allocation_test.cpp +33 -0
  29. data/vendor/datasketches-cpp/tdigest/include/tdigest_impl.hpp +1 -1
  30. data/vendor/datasketches-cpp/tdigest/test/tdigest_test.cpp +22 -0
  31. data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection.hpp +9 -0
  32. data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection_impl.hpp +14 -0
  33. data/vendor/datasketches-cpp/tuple/include/array_tuple_union.hpp +9 -0
  34. data/vendor/datasketches-cpp/tuple/include/array_tuple_union_impl.hpp +14 -0
  35. data/vendor/datasketches-cpp/tuple/test/array_of_doubles_sketch_test.cpp +47 -0
  36. data/vendor/datasketches-cpp/version.cfg.in +1 -1
  37. metadata +1 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6a91d4d694972af601b74f32fdec1d52fb48d6bf9ee0c16332219d3755afd4c6
4
- data.tar.gz: 5ca86cc02447be6af32c4eb1368fb93641cca227781a4e7827f2b1d3d43532bc
3
+ metadata.gz: 355d6f2a9ee22a1d96b01f87a655cc5c3f9421910eeb32fe24af92a23717ef54
4
+ data.tar.gz: c9ab5403080ced096259eb620dd41051a8ee62d5311568a14f2ff580c6ea53ed
5
5
  SHA512:
6
- metadata.gz: aaafc9cff65cd3827524ee7ae9d32076471adaaf8c17924121c1ca8f303cc76beafbf3fbaae4dea31895f31265b1a522795b3e6905a8275ba1eb3ae329032153
7
- data.tar.gz: 07ffe0606651871f0b10f4b31d383ef78db3c1fe2ae83021686e3ed835bb6f7437554ffe342f68f703bae799dae3176f391a0d3e642b42a620f160c079d5579d
6
+ metadata.gz: ce3dc862138b6e1fe9dafc30828dc8a4e79b5ec9891c3a3516d38abc75091b13478efd8a3992eb2366bacce582ad6784f5455adc9dd1e2bc3c9770fd7e493c5a
7
+ data.tar.gz: a4d6f0ee542ed5b7db7792d06e5a8e447dd60a95abea1e1dba7b1ce557de1dfef7349f2c4e3b814df0a56ca08c5d66bedc84405a4ffc666f003a9dfe4028913a
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 0.5.4 (2026-11-13)
2
+
3
+ - Updated DataSketches to 5.3.1
4
+
1
5
  ## 0.5.3 (2026-10-07)
2
6
 
3
7
  - Updated DataSketches to 5.3.0
@@ -1,3 +1,3 @@
1
1
  module DataSketches
2
- VERSION = "0.5.3"
2
+ VERSION = "0.5.4"
3
3
  end
@@ -214,40 +214,6 @@ APPENDIX B: Additional licenses relevant to this product.
214
214
  conditions of the following licenses.
215
215
 
216
216
 
217
- =============================================================
218
- Boost License (https://www.boost.org/LICENSE_1_0.txt)
219
- =============================================================
220
- Original source:
221
- https://github.com/catchorg/Catch2/blob/v2.x/LICENSE.txt
222
-
223
- Boost Software License - Version 1.0 - August 17th, 2003
224
-
225
- Permission is hereby granted, free of charge, to any person or organization
226
- obtaining a copy of the software and accompanying documentation covered by
227
- this license (the "Software") to use, reproduce, display, distribute,
228
- execute, and transmit the Software, and to prepare derivative works of the
229
- Software, and to permit third-parties to whom the Software is furnished to
230
- do so, all subject to the following:
231
-
232
- The copyright notices in the Software and this entire statement, including
233
- the above license grant, this restriction and the following disclaimer,
234
- must be included in all copies of the Software, in whole or in part, and
235
- all derivative works of the Software, unless such copies or derivative
236
- works are solely in the form of machine-executable object code generated by
237
- a source language processor.
238
-
239
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
240
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
241
- FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
242
- SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
243
- FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
244
- ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
245
- DEALINGS IN THE SOFTWARE.
246
- -------------------------------------------------------------
247
- Code Locations:
248
- Found in the Catch2 unit test code that is downloaded from github.com as part
249
- of CMake configuration if configured to build tests.
250
-
251
217
  =============================================================
252
218
  MIT License
253
219
  =============================================================
@@ -1,9 +1,9 @@
1
1
  Apache DataSketches C++
2
- Copyright 2025 The Apache Software Foundation
2
+ Copyright 2019-2026 The Apache Software Foundation
3
3
 
4
4
  Copyright 2015-2018 Yahoo Inc.
5
5
  Copyright 2019-2020 Verizon Media
6
- Copyright 2021- Yahoo Inc.
6
+ Copyright 2021-2025 Yahoo Inc.
7
7
 
8
8
  This product includes software developed at
9
9
  The Apache Software Foundation (http://www.apache.org/).
@@ -21,7 +21,7 @@ This code requires C++11.
21
21
 
22
22
  This library is header-only. The provided build process is only for unit tests.
23
23
 
24
- Building the unit tests requires CMake 3.12.0 or higher.
24
+ Building the unit tests requires CMake 3.16.0 or higher.
25
25
 
26
26
  Installing the latest CMake on OSX: `brew install cmake`.
27
27
 
@@ -44,13 +44,13 @@ static inline void check_memory_size(size_t requested_index, size_t capacity) {
44
44
 
45
45
  // note: size is in bytes, not items
46
46
  static inline size_t copy_from_mem(const void* src, void* dst, size_t size) {
47
- memcpy(dst, src, size);
47
+ if (size > 0) memcpy(dst, src, size);
48
48
  return size;
49
49
  }
50
50
 
51
51
  // note: size is in bytes, not items
52
52
  static inline size_t copy_to_mem(const void* src, void* dst, size_t size) {
53
- memcpy(dst, src, size);
53
+ if (size > 0) memcpy(dst, src, size);
54
54
  return size;
55
55
  }
56
56
 
@@ -382,7 +382,7 @@ private:
382
382
  std::vector<W, Allocator> _sketch_array; // the array stored by the sketch
383
383
  uint64_t _seed;
384
384
  W _total_weight;
385
- std::vector<uint64_t> hash_seeds;
385
+ std::vector<uint64_t, typename std::allocator_traits<Allocator>::template rebind_alloc<uint64_t>> hash_seeds;
386
386
 
387
387
  enum flags {IS_EMPTY};
388
388
  static const uint8_t PREAMBLE_LONGS_SHORT = 2; // Empty -> need second byte for sketch parameters
@@ -38,7 +38,8 @@ _num_hashes(num_hashes),
38
38
  _num_buckets(num_buckets),
39
39
  _sketch_array((static_cast<uint64_t>(num_hashes) * num_buckets < 1<<30) ? static_cast<size_t>(num_hashes) * num_buckets : 0, 0, _allocator),
40
40
  _seed(seed),
41
- _total_weight(0) {
41
+ _total_weight(0),
42
+ hash_seeds(_allocator) {
42
43
  if (num_buckets < 3) {
43
44
  throw std::invalid_argument("Using fewer than 3 buckets incurs relative error greater than 1.");
44
45
  }
@@ -152,4 +152,21 @@ TEST_CASE("CountMin sketch test allocator: bytes serialize-deserialize non-empty
152
152
  REQUIRE(test_allocator_net_allocations == 0);
153
153
  }
154
154
 
155
+ TEST_CASE("CountMin sketch test allocator: hash seeds use the sketch's allocator", "[cm_sketch_alloc]") {
156
+ test_allocator_total_bytes = 0;
157
+ test_allocator_net_allocations = 0;
158
+ {
159
+ const uint8_t n_hashes = 5;
160
+ const uint32_t n_buckets = 100;
161
+ const long long sketch_bytes = n_hashes * n_buckets * sizeof(uint64_t) + n_hashes * sizeof(uint64_t);
162
+ count_min_sketch_test_alloc c(n_hashes, n_buckets, DEFAULT_SEED, alloc(0));
163
+ // the counters and the hash seeds, all through the sketch's allocator
164
+ REQUIRE(test_allocator_total_bytes == sketch_bytes);
165
+ count_min_sketch_test_alloc copy(c);
166
+ REQUIRE(test_allocator_total_bytes == 2 * sketch_bytes);
167
+ }
168
+ REQUIRE(test_allocator_total_bytes == 0);
169
+ REQUIRE(test_allocator_net_allocations == 0);
170
+ }
171
+
155
172
  } // namespace datasketches
@@ -32,7 +32,9 @@ namespace datasketches {
32
32
 
33
33
  template<typename T>
34
34
  struct gaussian_kernel {
35
- T operator()(const std::vector<T>& v1, const std::vector<T>& v2) const {
35
+ // any two vectors of T, so the sketch's points can use its allocator
36
+ template<typename V1, typename V2>
37
+ T operator()(const V1& v1, const V2& v2) const {
36
38
  return exp(-std::inner_product(v1.begin(), v1.end(), v2.begin(), 0.0, std::plus<T>(), [](T a, T b){return (a-b)*(a-b);}));
37
39
  }
38
40
  };
@@ -203,7 +205,6 @@ private:
203
205
  static const uint8_t SERIAL_VERSION = 1;
204
206
  static const size_t LEVELS_ARRAY_START = 5;
205
207
 
206
- Allocator allocator_;
207
208
  Kernel kernel_;
208
209
  uint16_t k_;
209
210
  uint32_t dim_;
@@ -21,6 +21,7 @@
21
21
  #include <catch2/catch.hpp>
22
22
 
23
23
  #include <density_sketch.hpp>
24
+ #include <test_allocator.hpp>
24
25
 
25
26
  namespace datasketches {
26
27
 
@@ -241,4 +242,33 @@ TEST_CASE("serialize stream", "[density_sketch]") {
241
242
  }
242
243
  }
243
244
 
245
+ TEST_CASE("density sketch: custom allocator", "[density_sketch]") {
246
+ using alloc = test_allocator<double>;
247
+ using sketch = density_sketch<double, gaussian_kernel<double>, alloc>;
248
+ using point = std::vector<double, alloc>;
249
+ test_allocator_total_bytes = 0;
250
+ test_allocator_net_allocations = 0;
251
+ {
252
+ sketch s(10, 3, gaussian_kernel<double>(), alloc(0));
253
+ // Keep every retained point close to the query so the Gaussian kernel cannot underflow to zero.
254
+ for (int i = 0; i < 1000; ++i) s.update(point({i / 1000.0, 1.0, 2.0}, alloc(0)));
255
+ REQUIRE(s.get_n() == 1000);
256
+ REQUIRE(s.is_estimation_mode());
257
+ REQUIRE(s.get_estimate({0.5, 1.0, 2.0}) > 0);
258
+ REQUIRE(test_allocator_total_bytes > 0);
259
+
260
+ sketch other(10, 3, gaussian_kernel<double>(), alloc(0));
261
+ for (int i = 0; i < 100; ++i) other.update(point({i / 1000.0, 1.0, 2.0}, alloc(0)));
262
+ s.merge(other);
263
+ REQUIRE(s.get_n() == 1100);
264
+
265
+ auto bytes = s.serialize();
266
+ auto copy = sketch::deserialize(bytes.data(), bytes.size(), gaussian_kernel<double>(), alloc(0));
267
+ REQUIRE(copy.get_n() == s.get_n());
268
+ REQUIRE(copy.get_estimate({0.5, 1.0, 2.0}) == s.get_estimate({0.5, 1.0, 2.0}));
269
+ }
270
+ REQUIRE(test_allocator_total_bytes == 0);
271
+ REQUIRE(test_allocator_net_allocations == 0);
272
+ }
273
+
244
274
  } /* namespace datasketches */
@@ -54,7 +54,9 @@ void hll_union_alloc<A>::update(const hll_sketch_alloc<A>& sketch) {
54
54
  template<typename A>
55
55
  void hll_union_alloc<A>::update(hll_sketch_alloc<A>&& sketch) {
56
56
  if (sketch.is_empty()) { return; }
57
- if (gadget_.is_empty() && sketch.get_target_type() == HLL_8 && sketch.get_lg_config_k() <= lg_max_k_) {
57
+ // taking over the sketch is only possible if its memory can be released with the union's allocator
58
+ if (gadget_.is_empty() && sketch.get_target_type() == HLL_8 && sketch.get_lg_config_k() <= lg_max_k_
59
+ && sketch.sketch_impl->getAllocator() == gadget_.sketch_impl->getAllocator()) {
58
60
  if (sketch.get_current_mode() == HLL || sketch.get_lg_config_k() == lg_max_k_) {
59
61
  gadget_ = std::move(sketch);
60
62
  }
@@ -208,17 +210,20 @@ double hll_union_alloc<A>::get_rel_err(bool upper_bound, bool unioned,
208
210
  }
209
211
 
210
212
  template<typename A>
211
- HllSketchImpl<A>* hll_union_alloc<A>::copy_or_downsample(const HllSketchImpl<A>* src_impl, uint8_t tgt_lg_k) {
213
+ HllSketchImpl<A>* hll_union_alloc<A>::copy_or_downsample(const HllSketchImpl<A>* src_impl, uint8_t tgt_lg_k,
214
+ const A& allocator) {
212
215
  if (src_impl->getCurMode() != HLL) {
213
216
  throw std::logic_error("Attempt to downsample non-HLL sketch");
214
217
  }
215
218
  const HllArray<A>* src = static_cast<const HllArray<A>*>(src_impl);
216
- const int src_lg_k = src->getLgConfigK();
217
- if (src_lg_k <= tgt_lg_k) {
219
+ const uint8_t src_lg_k = src->getLgConfigK();
220
+ // a copy keeps the source's allocator, so it is only made if that is the union's allocator
221
+ if (src_lg_k <= tgt_lg_k && src->getAllocator() == allocator) {
218
222
  return src->copyAs(HLL_8);
219
223
  }
220
224
  typedef typename std::allocator_traits<A>::template rebind_alloc<Hll8Array<A>> hll8Alloc;
221
- Hll8Array<A>* tgtHllArr = new (hll8Alloc(src->getAllocator()).allocate(1)) Hll8Array<A>(tgt_lg_k, src->getAllocator());
225
+ const uint8_t lg_k = std::min(src_lg_k, tgt_lg_k);
226
+ Hll8Array<A>* tgtHllArr = new (hll8Alloc(allocator).allocate(1)) Hll8Array<A>(lg_k, allocator);
222
227
  tgtHllArr->mergeHll(*src);
223
228
  //both of these are required for isomorphism
224
229
  tgtHllArr->putHipAccum(src->getHipAccum());
@@ -239,8 +244,12 @@ template<typename A>
239
244
  void hll_union_alloc<A>::union_impl(const hll_sketch_alloc<A>& sketch, uint8_t lg_max_k) {
240
245
  const HllSketchImpl<A>* src_impl = sketch.sketch_impl; //default
241
246
  HllSketchImpl<A>* dst_impl = gadget_.sketch_impl; //default
247
+ // the gadget allocates with the union's allocator: the input sketch may use another allocator
248
+ // instance (e.g. a stateful one), and copies of it would keep that allocator
249
+ const A allocator = dst_impl->getAllocator();
242
250
  if (src_impl->getCurMode() == LIST || src_impl->getCurMode() == SET) {
243
- if (dst_impl->isEmpty() && src_impl->getLgConfigK() == dst_impl->getLgConfigK()) {
251
+ if (dst_impl->isEmpty() && src_impl->getLgConfigK() == dst_impl->getLgConfigK()
252
+ && src_impl->getAllocator() == allocator) {
244
253
  dst_impl = src_impl->copyAs(HLL_8);
245
254
  gadget_.sketch_impl->get_deleter()(gadget_.sketch_impl); // gadget to be replaced
246
255
  } else {
@@ -254,12 +263,12 @@ void hll_union_alloc<A>::union_impl(const hll_sketch_alloc<A>& sketch, uint8_t l
254
263
  // swap so that src is LIST or SET, tgt is HLL
255
264
  // use lg_max_k because LIST has effective K of 2^26
256
265
  const CouponList<A>* src = static_cast<const CouponList<A>*>(dst_impl);
257
- dst_impl = copy_or_downsample(src_impl, lg_max_k);
266
+ dst_impl = copy_or_downsample(src_impl, lg_max_k, allocator);
258
267
  static_cast<Hll8Array<A>*>(dst_impl)->mergeList(*src);
259
268
  gadget_.sketch_impl->get_deleter()(gadget_.sketch_impl); // gadget to be replaced
260
269
  } else { // gadget is HLL
261
270
  if (src_impl->getLgConfigK() < dst_impl->getLgConfigK()) {
262
- dst_impl = copy_or_downsample(dst_impl, sketch.get_lg_config_k());
271
+ dst_impl = copy_or_downsample(dst_impl, sketch.get_lg_config_k(), allocator);
263
272
  gadget_.sketch_impl->get_deleter()(gadget_.sketch_impl); // gadget to be replaced
264
273
  }
265
274
  const HllArray<A>* src = static_cast<const HllArray<A>*>(src_impl);
@@ -268,7 +277,7 @@ void hll_union_alloc<A>::union_impl(const hll_sketch_alloc<A>& sketch, uint8_t l
268
277
  static_cast<Hll8Array<A>*>(dst_impl)->putHipAccum(0);
269
278
  }
270
279
  } else { // src is HLL, gadget is empty
271
- dst_impl = copy_or_downsample(src_impl, lg_max_k);
280
+ dst_impl = copy_or_downsample(src_impl, lg_max_k, allocator);
272
281
  gadget_.sketch_impl->get_deleter()(gadget_.sketch_impl); // gadget to be replaced
273
282
  }
274
283
  gadget_.sketch_impl = dst_impl; // gadget replaced
@@ -640,7 +640,7 @@ class hll_union_alloc {
640
640
  */
641
641
  inline void union_impl(const hll_sketch_alloc<A>& sketch, uint8_t lg_max_k);
642
642
 
643
- static HllSketchImpl<A>* copy_or_downsample(const HllSketchImpl<A>* src_impl, uint8_t tgt_lg_k);
643
+ static HllSketchImpl<A>* copy_or_downsample(const HllSketchImpl<A>* src_impl, uint8_t tgt_lg_k, const A& allocator);
644
644
 
645
645
  void coupon_update(uint32_t coupon);
646
646
 
@@ -384,4 +384,77 @@ TEST_CASE("hll union: serialization is grouping independent", "[hll_union]") {
384
384
  REQUIRE(serialize_nested_union(a, b, c) == serialize_flat_union(a, b, c));
385
385
  }
386
386
 
387
+ // A stateful allocator: instances with different ids are different allocators, and the bytes each one
388
+ // has outstanding are counted.
389
+ static long long pool_bytes[2] = {0, 0};
390
+
391
+ template<typename T>
392
+ struct pool_allocator {
393
+ using value_type = T;
394
+ int id;
395
+ explicit pool_allocator(int id): id(id) {}
396
+ template<typename U>
397
+ pool_allocator(const pool_allocator<U>& other): id(other.id) {}
398
+ T* allocate(std::size_t n) {
399
+ pool_bytes[id] += n * sizeof(T);
400
+ return static_cast<T*>(::operator new(n * sizeof(T)));
401
+ }
402
+ void deallocate(T* p, std::size_t n) {
403
+ pool_bytes[id] -= n * sizeof(T);
404
+ ::operator delete(p);
405
+ }
406
+ };
407
+ template<typename T, typename U>
408
+ bool operator==(const pool_allocator<T>& a, const pool_allocator<U>& b) { return a.id == b.id; }
409
+ template<typename T, typename U>
410
+ bool operator!=(const pool_allocator<T>& a, const pool_allocator<U>& b) { return a.id != b.id; }
411
+
412
+ using pool_hll_sketch = hll_sketch_alloc<pool_allocator<uint8_t>>;
413
+ using pool_hll_union = hll_union_alloc<pool_allocator<uint8_t>>;
414
+
415
+ static pool_hll_sketch make_pool_sketch(int id, uint8_t lg_k, target_hll_type type, uint64_t n) {
416
+ pool_hll_sketch sketch(lg_k, type, false, pool_allocator<uint8_t>(id));
417
+ for (uint64_t i = 0; i < n; ++i) sketch.update(i);
418
+ return sketch;
419
+ }
420
+
421
+ TEST_CASE("hll union: keeps no memory of an input sketch's allocator", "[hll_union]") {
422
+ const uint8_t lg_max_k = 10;
423
+ // LIST, SET and HLL inputs, with a smaller, the same and a larger lg_k than the union
424
+ const uint64_t sizes[] = {5, 100, 10000};
425
+ const uint8_t lg_ks[] = {8, 10, 12};
426
+ for (uint64_t n: sizes) {
427
+ for (uint8_t lg_k: lg_ks) {
428
+ for (target_hll_type type: {HLL_4, HLL_6, HLL_8}) {
429
+ for (bool gadget_empty: {true, false}) {
430
+ for (bool move: {false, true}) {
431
+ hll_union u(lg_max_k);
432
+ pool_hll_union pool_union(lg_max_k, pool_allocator<uint8_t>(0));
433
+ if (!gadget_empty) {
434
+ for (uint64_t i = 0; i < 3 * n; ++i) {
435
+ u.update(i + n);
436
+ pool_union.update(i + n);
437
+ }
438
+ }
439
+ {
440
+ hll_sketch control(lg_k, type);
441
+ for (uint64_t i = 0; i < n; ++i) control.update(i);
442
+ u.update(control);
443
+ pool_hll_sketch input = make_pool_sketch(1, lg_k, type, n);
444
+ if (move) {
445
+ pool_union.update(std::move(input));
446
+ } else {
447
+ pool_union.update(input);
448
+ }
449
+ }
450
+ REQUIRE(pool_bytes[1] == 0);
451
+ REQUIRE(pool_union.get_estimate() == u.get_estimate());
452
+ }
453
+ }
454
+ }
455
+ }
456
+ }
457
+ REQUIRE(pool_bytes[0] == 0);
458
+ }
459
+
387
460
  } /* namespace datasketches */
@@ -591,10 +591,13 @@ class kll_sketch {
591
591
  uint32_t safe_level_size(uint8_t level) const;
592
592
  uint32_t get_num_retained_above_level_zero() const;
593
593
 
594
+ static void check_k(uint16_t k);
594
595
  static void check_m(uint8_t m);
595
596
  static void check_preamble_ints(uint8_t preamble_ints, uint8_t flags_byte);
596
597
  static void check_serial_version(uint8_t serial_version);
597
598
  static void check_family_id(uint8_t family_id);
599
+ static void check_num_levels(uint8_t num_levels);
600
+ static void check_levels(const vector_u32& levels);
598
601
 
599
602
  void check_sorting() const;
600
603
 
@@ -450,6 +450,7 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(std::istream& is, const Ser
450
450
  const auto m = read<uint8_t>(is);
451
451
  read<uint8_t>(is); // skip unused byte
452
452
 
453
+ check_k(k);
453
454
  check_m(m);
454
455
  check_preamble_ints(preamble_ints, flags_byte);
455
456
  check_serial_version(serial_version);
@@ -472,6 +473,7 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(std::istream& is, const Ser
472
473
  min_k = read<uint16_t>(is);
473
474
  num_levels = read<uint8_t>(is);
474
475
  read<uint8_t>(is); // skip unused byte
476
+ check_num_levels(num_levels);
475
477
  }
476
478
  vector_u32 levels(num_levels + 1, 0, allocator);
477
479
  const uint32_t capacity(kll_helper::compute_total_capacity(k, m, num_levels));
@@ -480,8 +482,10 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(std::istream& is, const Ser
480
482
  } else {
481
483
  // the last integer in levels_ is not serialized because it can be derived
482
484
  read(is, levels.data(), sizeof(levels[0]) * num_levels);
485
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
483
486
  }
484
487
  levels[num_levels] = capacity;
488
+ check_levels(levels);
485
489
  optional<T> min_item;
486
490
  optional<T> max_item;
487
491
  if (!is_single_item) {
@@ -537,6 +541,7 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(const void* bytes, size_t s
537
541
  ptr += copy_from_mem(ptr, m);
538
542
  ptr += sizeof(uint8_t); // skip unused byte
539
543
 
544
+ check_k(k);
540
545
  check_m(m);
541
546
  check_preamble_ints(preamble_ints, flags_byte);
542
547
  check_serial_version(serial_version);
@@ -560,6 +565,7 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(const void* bytes, size_t s
560
565
  ptr += copy_from_mem(ptr, min_k);
561
566
  ptr += copy_from_mem(ptr, num_levels);
562
567
  ptr += sizeof(uint8_t); // skip unused byte
568
+ check_num_levels(num_levels);
563
569
  }
564
570
  vector_u32 levels(num_levels + 1, 0, allocator);
565
571
  const uint32_t capacity(kll_helper::compute_total_capacity(k, m, num_levels));
@@ -567,9 +573,11 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(const void* bytes, size_t s
567
573
  levels[0] = capacity - 1;
568
574
  } else {
569
575
  // the last integer in levels_ is not serialized because it can be derived
576
+ ensure_minimum_memory(size, (ptr - static_cast<const char*>(bytes)) + sizeof(levels[0]) * num_levels);
570
577
  ptr += copy_from_mem(ptr, levels.data(), sizeof(levels[0]) * num_levels);
571
578
  }
572
579
  levels[num_levels] = capacity;
580
+ check_levels(levels);
573
581
  optional<T> min_item;
574
582
  optional<T> max_item;
575
583
  if (!is_single_item) {
@@ -867,6 +875,14 @@ uint32_t kll_sketch<T, C, A>::get_num_retained_above_level_zero() const {
867
875
  return levels_[num_levels_] - levels_[1];
868
876
  }
869
877
 
878
+ template<typename T, typename C, typename A>
879
+ void kll_sketch<T, C, A>::check_k(uint16_t k) {
880
+ if (k < kll_constants::MIN_K) {
881
+ throw std::invalid_argument("Possible corruption: K must be >= " + std::to_string(kll_constants::MIN_K)
882
+ + ": " + std::to_string(k));
883
+ }
884
+ }
885
+
870
886
  template<typename T, typename C, typename A>
871
887
  void kll_sketch<T, C, A>::check_m(uint8_t m) {
872
888
  if (m != kll_constants::DEFAULT_M) {
@@ -909,6 +925,25 @@ void kll_sketch<T, C, A>::check_family_id(uint8_t family_id) {
909
925
  }
910
926
  }
911
927
 
928
+ template<typename T, typename C, typename A>
929
+ void kll_sketch<T, C, A>::check_num_levels(uint8_t num_levels) {
930
+ if (num_levels == 0) {
931
+ throw std::invalid_argument("Possible corruption: number of levels must be > 0");
932
+ }
933
+ }
934
+
935
+ // The last entry is the capacity. Each level must start at or after the previous one,
936
+ // so every level lies within the items buffer.
937
+ template<typename T, typename C, typename A>
938
+ void kll_sketch<T, C, A>::check_levels(const vector_u32& levels) {
939
+ for (size_t i = 1; i < levels.size(); ++i) {
940
+ if (levels[i - 1] > levels[i]) {
941
+ throw std::invalid_argument("Possible corruption: levels[" + std::to_string(i - 1) + "] = "
942
+ + std::to_string(levels[i - 1]) + " > levels[" + std::to_string(i) + "] = " + std::to_string(levels[i]));
943
+ }
944
+ }
945
+ }
946
+
912
947
  template <typename T, typename C, typename A>
913
948
  string<A> kll_sketch<T, C, A>::to_string(bool print_levels, bool print_items) const {
914
949
  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
@@ -835,4 +835,74 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
835
835
  REQUIRE(test_allocator_total_bytes == 0);
836
836
  }
837
837
 
838
+
839
+ TEST_CASE("kll sketch: deserialize rejects inconsistent levels", "[kll_sketch]") {
840
+ using vector_bytes = kll_sketch<int>::vector_bytes;
841
+ const size_t k_offset = 4;
842
+ const size_t num_levels_offset = 18;
843
+ const size_t levels_offset = 20;
844
+ auto put_u32 = [](vector_bytes& bytes, size_t offset, uint32_t value) {
845
+ std::memcpy(bytes.data() + offset, &value, sizeof(value));
846
+ };
847
+ auto get_u32 = [](const vector_bytes& bytes, size_t offset) {
848
+ uint32_t value;
849
+ std::memcpy(&value, bytes.data() + offset, sizeof(value));
850
+ return value;
851
+ };
852
+ auto check_rejected = [](const vector_bytes& bytes) {
853
+ REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
854
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
855
+ s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
856
+ REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(s), std::invalid_argument);
857
+ };
858
+
859
+ kll_sketch<int> one_level(8);
860
+ for (int i = 0; i < 3; ++i) one_level.update(i);
861
+ kll_sketch<int> three_levels(8);
862
+ for (int i = 0; i < 40; ++i) three_levels.update(i);
863
+ REQUIRE(three_levels.serialize()[num_levels_offset] == 3);
864
+
865
+ SECTION("first level beyond capacity") {
866
+ auto bytes = one_level.serialize();
867
+ put_u32(bytes, levels_offset, 1009);
868
+ check_rejected(bytes);
869
+ }
870
+
871
+ SECTION("level beyond capacity") {
872
+ auto bytes = three_levels.serialize();
873
+ put_u32(bytes, levels_offset + sizeof(uint32_t), 100000);
874
+ check_rejected(bytes);
875
+ }
876
+
877
+ SECTION("decreasing levels") {
878
+ auto bytes = three_levels.serialize();
879
+ REQUIRE(get_u32(bytes, levels_offset) > 0);
880
+ put_u32(bytes, levels_offset + sizeof(uint32_t), 0);
881
+ check_rejected(bytes);
882
+ }
883
+
884
+ SECTION("zero levels") {
885
+ auto bytes = three_levels.serialize();
886
+ bytes[num_levels_offset] = 0;
887
+ check_rejected(bytes);
888
+ }
889
+
890
+ SECTION("k below minimum") {
891
+ auto bytes = three_levels.serialize();
892
+ const uint16_t k = 4;
893
+ std::memcpy(bytes.data() + k_offset, &k, sizeof(k));
894
+ check_rejected(bytes);
895
+ }
896
+
897
+ SECTION("levels beyond the input") {
898
+ auto bytes = three_levels.serialize();
899
+ bytes[num_levels_offset] = 40;
900
+ bytes.resize(levels_offset);
901
+ REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(bytes.data(), bytes.size()), std::out_of_range);
902
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
903
+ s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
904
+ REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(s), std::runtime_error);
905
+ }
906
+ }
907
+
838
908
  } /* namespace datasketches */
@@ -531,6 +531,7 @@ auto quantiles_sketch<T, C, A>::deserialize(const void* bytes, size_t size, cons
531
531
 
532
532
  if (serial_version == 1) {
533
533
  uint64_t unused_long;
534
+ ensure_minimum_memory(end_ptr - ptr, sizeof(unused_long));
534
535
  ptr += copy_from_mem(ptr, unused_long); // no longer used
535
536
  }
536
537
 
@@ -956,4 +956,17 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
956
956
  }
957
957
  }
958
958
 
959
+
960
+ TEST_CASE("quantiles sketch: deserialize serial version 1 truncated after min and max", "[quantiles_sketch]") {
961
+ quantiles_sketch<int> sketch(16);
962
+ for (int i = 0; i < 1000; ++i) sketch.update(i);
963
+ auto bytes = sketch.serialize();
964
+ // serial version 1 has an unused 8-byte field after min and max
965
+ bytes[0] = 5; // preamble longs
966
+ bytes[1] = 1; // serial version
967
+ bytes[3] &= ~(1 << 3); // not compact
968
+ bytes.resize(8 + sizeof(uint64_t) + 2 * sizeof(int)); // preamble, n, min and max
969
+ REQUIRE_THROWS_AS(quantiles_sketch<int>::deserialize(bytes.data(), bytes.size()), std::out_of_range);
970
+ }
971
+
959
972
  } /* namespace datasketches */
@@ -31,6 +31,8 @@ namespace datasketches {
31
31
  namespace req_constants {
32
32
  /// minimum value of parameter K
33
33
  const uint16_t MIN_K = 4;
34
+ /// maximum value of parameter K
35
+ const uint16_t MAX_K = 1024;
34
36
  /// initial number of sections
35
37
  const uint8_t INIT_NUM_SECTIONS = 3;
36
38
  /// multiplier for nominal capacity
@@ -87,11 +87,11 @@ public:
87
87
 
88
88
  template<typename S>
89
89
  static req_compactor deserialize(std::istream& is, const S& serde, const Comparator& comparator,
90
- const Allocator& allocator, bool sorted, bool hra);
90
+ const Allocator& allocator, bool sorted, bool hra, uint16_t k, uint8_t level);
91
91
 
92
92
  template<typename S>
93
93
  static std::pair<req_compactor, size_t> deserialize(const void* bytes, size_t size, const S& serde,
94
- const Comparator& comparator, const Allocator& allocator, bool sorted, bool hra);
94
+ const Comparator& comparator, const Allocator& allocator, bool sorted, bool hra, uint16_t k, uint8_t level);
95
95
 
96
96
  template<typename S>
97
97
  static req_compactor deserialize(std::istream& is, const S& serde, const Comparator& comparator,
@@ -122,6 +122,7 @@ private:
122
122
  void ensure_space(uint32_t num);
123
123
 
124
124
  static uint32_t nearest_even(float value);
125
+ static void check_header(uint16_t k, uint8_t level, uint8_t lg_weight, float section_size_raw, uint8_t num_sections);
125
126
 
126
127
  template<typename InIter, typename OutIter>
127
128
  static void promote_evens_or_odds(InIter from, InIter to, bool flag, OutIter dst);