datasketches 0.5.3 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/lib/datasketches/version.rb +1 -1
  4. data/vendor/datasketches-cpp/LICENSE +0 -34
  5. data/vendor/datasketches-cpp/NOTICE +2 -2
  6. data/vendor/datasketches-cpp/README.md +1 -1
  7. data/vendor/datasketches-cpp/common/include/memory_operations.hpp +2 -2
  8. data/vendor/datasketches-cpp/count/include/count_min.hpp +1 -1
  9. data/vendor/datasketches-cpp/count/include/count_min_impl.hpp +2 -1
  10. data/vendor/datasketches-cpp/count/test/count_min_allocation_test.cpp +17 -0
  11. data/vendor/datasketches-cpp/density/include/density_sketch.hpp +3 -2
  12. data/vendor/datasketches-cpp/density/test/density_sketch_test.cpp +30 -0
  13. data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp +18 -9
  14. data/vendor/datasketches-cpp/hll/include/hll.hpp +1 -1
  15. data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +73 -0
  16. data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +3 -0
  17. data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +35 -0
  18. data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +70 -0
  19. data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch_impl.hpp +1 -0
  20. data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_test.cpp +13 -0
  21. data/vendor/datasketches-cpp/req/include/req_common.hpp +2 -0
  22. data/vendor/datasketches-cpp/req/include/req_compactor.hpp +3 -2
  23. data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +31 -3
  24. data/vendor/datasketches-cpp/req/include/req_sketch.hpp +5 -1
  25. data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +31 -3
  26. data/vendor/datasketches-cpp/req/test/req_sketch_test.cpp +176 -0
  27. data/vendor/datasketches-cpp/sampling/include/ebpps_sketch_impl.hpp +1 -1
  28. data/vendor/datasketches-cpp/sampling/test/ebpps_allocation_test.cpp +33 -0
  29. data/vendor/datasketches-cpp/tdigest/include/tdigest_impl.hpp +1 -1
  30. data/vendor/datasketches-cpp/tdigest/test/tdigest_test.cpp +22 -0
  31. data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection.hpp +9 -0
  32. data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection_impl.hpp +14 -0
  33. data/vendor/datasketches-cpp/tuple/include/array_tuple_union.hpp +9 -0
  34. data/vendor/datasketches-cpp/tuple/include/array_tuple_union_impl.hpp +14 -0
  35. data/vendor/datasketches-cpp/tuple/test/array_of_doubles_sketch_test.cpp +47 -0
  36. data/vendor/datasketches-cpp/version.cfg.in +1 -1
  37. metadata +1 -1
@@ -326,6 +326,30 @@ uint32_t req_compactor<T, C, A>::nearest_even(float value) {
326
326
  return static_cast<uint32_t>(round(value / 2)) << 1;
327
327
  }
328
328
 
329
+ template<typename T, typename C, typename A>
330
+ void req_compactor<T, C, A>::check_header(uint16_t k, uint8_t level, uint8_t lg_weight, float section_size_raw,
331
+ uint8_t num_sections) {
332
+ // the sketch creates the compactor at level h with lg_weight h
333
+ if (lg_weight != level) {
334
+ throw std::invalid_argument("Possible corruption: compactor " + std::to_string(level) + " has lg_weight "
335
+ + std::to_string(lg_weight));
336
+ }
337
+ // the section size starts at k and shrinks to no less than MIN_K
338
+ // (the first check is false for NaN and keeps nearest_even() in range)
339
+ const uint32_t section_size = (section_size_raw > 0 && section_size_raw <= k + 1) ? nearest_even(section_size_raw) : 0;
340
+ if (section_size < req_constants::MIN_K || section_size > k) {
341
+ throw std::invalid_argument("Possible corruption: section size " + std::to_string(section_size_raw)
342
+ + " is out of range for k " + std::to_string(k));
343
+ }
344
+ // the number of sections starts at 3 and doubles, and must keep 1 << (num_sections - 1) defined
345
+ // the limit uses the raw section size: when that has shrunk to 3, the even section size is 4 and can exceed it
346
+ const uint32_t rounded_section_size = static_cast<uint32_t>(round(section_size_raw));
347
+ if (num_sections < 1 || num_sections > 64 || rounded_section_size * num_sections > static_cast<uint32_t>(k) * k) {
348
+ throw std::invalid_argument("Possible corruption: number of sections " + std::to_string(num_sections)
349
+ + " is out of range for section size " + std::to_string(section_size_raw) + " and k " + std::to_string(k));
350
+ }
351
+ }
352
+
329
353
  template<typename T, typename C, typename A>
330
354
  template<typename InIter, typename OutIter>
331
355
  void req_compactor<T, C, A>::promote_evens_or_odds(InIter from, InIter to, bool odds, OutIter dst) {
@@ -394,13 +418,15 @@ size_t req_compactor<T, C, A>::serialize(void* dst, size_t capacity, const S& se
394
418
  template<typename T, typename C, typename A>
395
419
  template<typename S>
396
420
  req_compactor<T, C, A> req_compactor<T, C, A>::deserialize(std::istream& is, const S& serde,
397
- const C& comparator, const A& allocator, bool sorted, bool hra) {
421
+ const C& comparator, const A& allocator, bool sorted, bool hra, uint16_t k, uint8_t level) {
398
422
  auto state = read<decltype(state_)>(is);
399
423
  auto section_size_raw = read<decltype(section_size_raw_)>(is);
400
424
  auto lg_weight = read<decltype(lg_weight_)>(is);
401
425
  auto num_sections = read<decltype(num_sections_)>(is);
402
426
  read<uint16_t>(is); // padding
403
427
  auto num_items = read<uint32_t>(is);
428
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
429
+ check_header(k, level, lg_weight, section_size_raw, num_sections);
404
430
  auto items = deserialize_items(is, serde, allocator, num_items);
405
431
  return req_compactor(hra, lg_weight, sorted, section_size_raw, num_sections, state, std::move(items), num_items,
406
432
  comparator, allocator);
@@ -431,8 +457,9 @@ auto req_compactor<T, C, A>::deserialize_items(std::istream& is, const S& serde,
431
457
  template<typename T, typename C, typename A>
432
458
  template<typename S>
433
459
  std::pair<req_compactor<T, C, A>, size_t> req_compactor<T, C, A>::deserialize(const void* bytes, size_t size,
434
- const S& serde, const C& comparator, const A& allocator, bool sorted, bool hra) {
435
- ensure_minimum_memory(size, 8);
460
+ const S& serde, const C& comparator, const A& allocator, bool sorted, bool hra, uint16_t k, uint8_t level) {
461
+ ensure_minimum_memory(size, sizeof(state_) + sizeof(section_size_raw_) + sizeof(lg_weight_) + sizeof(num_sections_)
462
+ + sizeof(uint16_t) + sizeof(num_items_));
436
463
  const char* ptr = static_cast<const char*>(bytes);
437
464
  const char* end_ptr = static_cast<const char*>(bytes) + size;
438
465
 
@@ -447,6 +474,7 @@ std::pair<req_compactor<T, C, A>, size_t> req_compactor<T, C, A>::deserialize(co
447
474
  ptr += 2; // padding
448
475
  uint32_t num_items;
449
476
  ptr += copy_from_mem(ptr, num_items);
477
+ check_header(k, level, lg_weight, section_size_raw, num_sections);
450
478
  auto pair = deserialize_items(ptr, end_ptr - ptr, serde, allocator, num_items);
451
479
  ptr += pair.second;
452
480
  return std::pair<req_compactor, size_t>(
@@ -98,12 +98,14 @@ public:
98
98
 
99
99
  /**
100
100
  * Constructor
101
- * @param k Controls the size and error of the sketch. It must be even and in the range [4, 1024], inclusive.
101
+ * @param k Controls the size and error of the sketch. It must be no greater than 1024.
102
+ * Values below 4 are raised to 4, and odd values are rounded down to the nearest even value.
102
103
  * Value of 12 roughly corresponds to 1% relative error guarantee at 95% confidence.
103
104
  * @param hra if true, the default, the high ranks are prioritized for better
104
105
  * accuracy. Otherwise the low ranks are prioritized for better accuracy.
105
106
  * @param comparator strict weak ordering function (see C++ named requirements: Compare)
106
107
  * @param allocator used by this sketch to allocate memory
108
+ * @throws std::invalid_argument if k is greater than 1024
107
109
  */
108
110
  explicit req_sketch(uint16_t k, bool hra = true, const Comparator& comparator = Comparator(),
109
111
  const Allocator& allocator = Allocator());
@@ -463,7 +465,9 @@ private:
463
465
  optional<T>&& min_item, optional<T>&& max_item,
464
466
  std::vector<Compactor, AllocCompactor>&& compactors, const Comparator& comparator);
465
467
 
468
+ static void check_k(uint16_t k);
466
469
  static void check_preamble_ints(uint8_t preamble_ints, uint8_t num_levels);
470
+ static void check_num_levels(uint8_t num_levels, bool raw_items);
467
471
  static void check_serial_version(uint8_t serial_version);
468
472
  static void check_family_id(uint8_t family_id);
469
473
 
@@ -30,7 +30,7 @@ template<typename T, typename C, typename A>
30
30
  req_sketch<T, C, A>::req_sketch(uint16_t k, bool hra, const C& comparator, const A& allocator):
31
31
  comparator_(comparator),
32
32
  allocator_(allocator),
33
- k_(std::max<uint8_t>(static_cast<int>(k) & -2, static_cast<int>(req_constants::MIN_K))), //rounds down one if odd
33
+ k_(std::max<uint16_t>(static_cast<int>(k) & -2, static_cast<int>(req_constants::MIN_K))), //rounds down one if odd
34
34
  hra_(hra),
35
35
  max_nom_size_(0),
36
36
  num_retained_(0),
@@ -40,6 +40,9 @@ min_item_(),
40
40
  max_item_(),
41
41
  sorted_view_(nullptr)
42
42
  {
43
+ if (k > req_constants::MAX_K) {
44
+ throw std::invalid_argument("K must be <= " + std::to_string(req_constants::MAX_K) + ": " + std::to_string(k));
45
+ }
43
46
  grow();
44
47
  }
45
48
 
@@ -453,6 +456,7 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(std::istream& is, const Ser
453
456
  const auto num_levels = read<uint8_t>(is);
454
457
  const auto num_raw_items = read<uint8_t>(is);
455
458
 
459
+ check_k(k);
456
460
  check_preamble_ints(preamble_ints, num_levels);
457
461
  check_serial_version(serial_version);
458
462
  check_family_id(family_id);
@@ -466,6 +470,7 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(std::istream& is, const Ser
466
470
  optional<T> max_item;
467
471
 
468
472
  const bool raw_items = flags_byte & (1 << flags::RAW_ITEMS);
473
+ check_num_levels(num_levels, raw_items);
469
474
  const bool is_level_0_sorted = flags_byte & (1 << flags::IS_LEVEL_ZERO_SORTED);
470
475
  std::vector<Compactor, AllocCompactor> compactors(allocator);
471
476
 
@@ -491,10 +496,12 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(std::istream& is, const Ser
491
496
  compactors.push_back(Compactor::deserialize(is, sd, comparator, allocator, is_level_0_sorted, k, num_raw_items, hra));
492
497
  } else {
493
498
  for (size_t i = 0; i < num_levels; ++i) {
494
- compactors.push_back(Compactor::deserialize(is, sd, comparator, allocator, i == 0 ? is_level_0_sorted : true, hra));
499
+ compactors.push_back(Compactor::deserialize(is, sd, comparator, allocator, i == 0 ? is_level_0_sorted : true, hra,
500
+ k, static_cast<uint8_t>(i)));
495
501
  }
496
502
  }
497
503
  if (num_levels == 1) {
504
+ if (compactors[0].get_num_items() == 0) throw std::invalid_argument("Possible corruption: non-empty sketch with no items");
498
505
  const auto begin = compactors[0].begin();
499
506
  const auto end = compactors[0].end();
500
507
  n = compactors[0].get_num_items();
@@ -534,6 +541,7 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(const void* bytes, size_t s
534
541
  uint8_t num_raw_items;
535
542
  ptr += copy_from_mem(ptr, num_raw_items);
536
543
 
544
+ check_k(k);
537
545
  check_preamble_ints(preamble_ints, num_levels);
538
546
  check_serial_version(serial_version);
539
547
  check_family_id(family_id);
@@ -546,6 +554,7 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(const void* bytes, size_t s
546
554
  optional<T> max_item;
547
555
 
548
556
  const bool raw_items = flags_byte & (1 << flags::RAW_ITEMS);
557
+ check_num_levels(num_levels, raw_items);
549
558
  const bool is_level_0_sorted = flags_byte & (1 << flags::IS_LEVEL_ZERO_SORTED);
550
559
  std::vector<Compactor, AllocCompactor> compactors(allocator);
551
560
 
@@ -574,12 +583,14 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(const void* bytes, size_t s
574
583
  ptr += pair.second;
575
584
  } else {
576
585
  for (size_t i = 0; i < num_levels; ++i) {
577
- auto pair = Compactor::deserialize(ptr, end_ptr - ptr, sd, comparator, allocator, i == 0 ? is_level_0_sorted : true, hra);
586
+ auto pair = Compactor::deserialize(ptr, end_ptr - ptr, sd, comparator, allocator, i == 0 ? is_level_0_sorted : true, hra,
587
+ k, static_cast<uint8_t>(i));
578
588
  compactors.push_back(std::move(pair.first));
579
589
  ptr += pair.second;
580
590
  }
581
591
  }
582
592
  if (num_levels == 1) {
593
+ if (compactors[0].get_num_items() == 0) throw std::invalid_argument("Possible corruption: non-empty sketch with no items");
583
594
  const auto begin = compactors[0].begin();
584
595
  const auto end = compactors[0].end();
585
596
  n = compactors[0].get_num_items();
@@ -703,6 +714,23 @@ sorted_view_(nullptr)
703
714
  update_num_retained();
704
715
  }
705
716
 
717
+ template<typename T, typename C, typename A>
718
+ void req_sketch<T, C, A>::check_k(uint16_t k) {
719
+ if (k < req_constants::MIN_K || k > req_constants::MAX_K || (k & 1) != 0) {
720
+ throw std::invalid_argument("Possible corruption: k must be even and in the range ["
721
+ + std::to_string(req_constants::MIN_K) + ", " + std::to_string(req_constants::MAX_K) + "]: " + std::to_string(k));
722
+ }
723
+ }
724
+
725
+ template<typename T, typename C, typename A>
726
+ void req_sketch<T, C, A>::check_num_levels(uint8_t num_levels, bool raw_items) {
727
+ // a non-empty sketch has at least one compactor, and raw items fit in one
728
+ if (num_levels == 0 || (raw_items && num_levels != 1)) {
729
+ throw std::invalid_argument("Possible corruption: number of levels " + std::to_string(num_levels)
730
+ + (raw_items ? " with raw items" : ""));
731
+ }
732
+ }
733
+
706
734
  template<typename T, typename C, typename A>
707
735
  void req_sketch<T, C, A>::check_preamble_ints(uint8_t preamble_ints, uint8_t num_levels) {
708
736
  const uint8_t expected_preamble_ints = num_levels > 1 ? 4 : 2;
@@ -23,6 +23,7 @@
23
23
 
24
24
  #include <fstream>
25
25
  #include <sstream>
26
+ #include <cstring>
26
27
  #include <limits>
27
28
  #include <stdexcept>
28
29
 
@@ -54,6 +55,39 @@ TEST_CASE("req sketch: empty", "[req_sketch]") {
54
55
  REQUIRE_THROWS_AS(sketch.get_PMF(split_points, 1), std::runtime_error);
55
56
  }
56
57
 
58
+ TEST_CASE("req sketch: k above 255", "[req_sketch]") {
59
+ // k used to be truncated to 8 bits: 256 became 4 and 500 became 244
60
+ REQUIRE(req_sketch<float>(256).get_k() == 256);
61
+ REQUIRE(req_sketch<float>(500).get_k() == 500);
62
+ REQUIRE(req_sketch<float>(1024).get_k() == 1024);
63
+ REQUIRE(req_sketch<float>(1001).get_k() == 1000); // rounds down if odd
64
+
65
+ req_sketch<float> sketch(1024);
66
+ const size_t n = 100000;
67
+ for (size_t i = 0; i < n; i++) sketch.update(static_cast<float>(i));
68
+ REQUIRE(sketch.is_estimation_mode());
69
+ auto bytes = sketch.serialize();
70
+ auto sketch2 = req_sketch<float>::deserialize(bytes.data(), bytes.size());
71
+ REQUIRE(sketch2.get_k() == 1024);
72
+ REQUIRE(sketch2.get_n() == n);
73
+ REQUIRE(sketch2.get_num_retained() == sketch.get_num_retained());
74
+ }
75
+
76
+ TEST_CASE("req sketch: k bounds", "[req_sketch]") {
77
+ for (bool hra: {true, false}) {
78
+ REQUIRE(req_sketch<float>(0, hra).get_k() == 4);
79
+ REQUIRE(req_sketch<float>(3, hra).get_k() == 4);
80
+ REQUIRE(req_sketch<float>(4, hra).get_k() == 4);
81
+ REQUIRE(req_sketch<float>(5, hra).get_k() == 4);
82
+ REQUIRE(req_sketch<float>(1023, hra).get_k() == 1022);
83
+ REQUIRE(req_sketch<float>(1024, hra).get_k() == 1024);
84
+ // Reject the original k, even if rounding down would put it in range.
85
+ REQUIRE_THROWS_AS(req_sketch<float>(1025, hra), std::invalid_argument);
86
+ REQUIRE_THROWS_AS(req_sketch<float>(1026, hra), std::invalid_argument);
87
+ REQUIRE_THROWS_AS(req_sketch<float>(std::numeric_limits<uint16_t>::max(), hra), std::invalid_argument);
88
+ }
89
+ }
90
+
57
91
  TEST_CASE("req sketch: single value, lra", "[req_sketch]") {
58
92
  req_sketch<float> sketch(12, false);
59
93
  sketch.update(1.0f);
@@ -559,4 +593,146 @@ TEST_CASE("get_rank equivalence") {
559
593
  // sketch.serialize(os);
560
594
  //}
561
595
 
596
+
597
+ template<typename Bytes, typename V>
598
+ static void put_value(Bytes& bytes, size_t offset, V value) {
599
+ std::memcpy(bytes.data() + offset, &value, sizeof(value));
600
+ }
601
+
602
+ TEST_CASE("req sketch: deserialize rejects inconsistent compactors", "[req_sketch]") {
603
+ using vector_bytes = req_sketch<float>::vector_bytes;
604
+ // preamble: k at 4, number of levels at 6, number of raw items at 7
605
+ // compactor: state(8), section size(4), lg_weight(1), number of sections(1), padding(2), number of items(4), items
606
+ const size_t k_offset = 4;
607
+ const size_t num_levels_offset = 6;
608
+ const size_t num_raw_items_offset = 7;
609
+ const size_t estimation_compactor = 24; // after the preamble, n, min and max
610
+ const size_t exact_compactor = 8; // after the preamble
611
+ const size_t section_size = 8;
612
+ const size_t lg_weight = 12;
613
+ const size_t num_sections = 13;
614
+ const size_t num_items = 16;
615
+ const size_t items = 20;
616
+ auto check_rejected = [](const vector_bytes& bytes) {
617
+ REQUIRE_THROWS_AS(req_sketch<float>::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
618
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
619
+ s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
620
+ REQUIRE_THROWS_AS(req_sketch<float>::deserialize(s), std::invalid_argument);
621
+ };
622
+
623
+ req_sketch<float> raw(12);
624
+ for (int i = 0; i < 3; ++i) raw.update(static_cast<float>(i));
625
+ req_sketch<float> exact(12);
626
+ for (int i = 0; i < 50; ++i) exact.update(static_cast<float>(i));
627
+ req_sketch<float> estimation(12);
628
+ for (int i = 0; i < 1000; ++i) estimation.update(static_cast<float>(i));
629
+ REQUIRE(estimation.serialize()[num_levels_offset] > 2);
630
+
631
+ SECTION("valid images") {
632
+ // sketches deep enough for the sections to have grown, in both rank accuracy modes
633
+ for (bool hra: {true, false}) {
634
+ req_sketch<float> sketch(12, hra);
635
+ for (int i = 0; i < 1000000; ++i) sketch.update(static_cast<float>(i));
636
+ auto bytes = sketch.serialize();
637
+ REQUIRE(req_sketch<float>::deserialize(bytes.data(), bytes.size()).get_n() == sketch.get_n());
638
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
639
+ sketch.serialize(s);
640
+ REQUIRE(req_sketch<float>::deserialize(s).get_n() == sketch.get_n());
641
+ }
642
+ }
643
+
644
+ SECTION("valid images with the smallest section size") {
645
+ // the raw section size has shrunk to 3, which the section size rounds up to 4
646
+ req_sketch<float> sketch(6);
647
+ for (int i = 0; i < 100000; ++i) sketch.update(static_cast<float>(i));
648
+ auto bytes = sketch.serialize();
649
+ float raw_section_size;
650
+ std::memcpy(&raw_section_size, bytes.data() + estimation_compactor + section_size, sizeof(raw_section_size));
651
+ REQUIRE(raw_section_size == Approx(3).margin(0.001));
652
+ REQUIRE(bytes[estimation_compactor + num_sections] == 12);
653
+ REQUIRE(req_sketch<float>::deserialize(bytes.data(), bytes.size()).get_n() == sketch.get_n());
654
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
655
+ sketch.serialize(s);
656
+ REQUIRE(req_sketch<float>::deserialize(s).get_n() == sketch.get_n());
657
+
658
+ // k = 12 needs about 2 billion items to get here, so set the header it would have
659
+ bytes = estimation.serialize();
660
+ put_value(bytes, estimation_compactor + section_size, 3.0f);
661
+ put_value(bytes, estimation_compactor + num_sections, static_cast<uint8_t>(48));
662
+ REQUIRE(req_sketch<float>::deserialize(bytes.data(), bytes.size()).get_n() == estimation.get_n());
663
+ }
664
+
665
+ SECTION("invalid k") {
666
+ for (uint16_t k: {2, 13, 2000}) {
667
+ auto bytes = estimation.serialize();
668
+ put_value(bytes, k_offset, k);
669
+ check_rejected(bytes);
670
+ }
671
+ }
672
+
673
+ SECTION("section size out of range") {
674
+ for (float size: {0.0f, 2.0f, 14.0f, 1e12f, std::numeric_limits<float>::quiet_NaN()}) {
675
+ auto bytes = estimation.serialize();
676
+ put_value(bytes, estimation_compactor + section_size, size);
677
+ check_rejected(bytes);
678
+ }
679
+ }
680
+
681
+ SECTION("number of sections out of range") {
682
+ for (uint8_t sections: {0, 65}) {
683
+ auto bytes = estimation.serialize();
684
+ put_value(bytes, estimation_compactor + num_sections, sections);
685
+ check_rejected(bytes);
686
+ }
687
+ // in range, but too many for this section size and k
688
+ auto bytes = estimation.serialize();
689
+ put_value(bytes, estimation_compactor + section_size, 3.0f);
690
+ put_value(bytes, estimation_compactor + num_sections, static_cast<uint8_t>(64));
691
+ check_rejected(bytes);
692
+ }
693
+
694
+ SECTION("wrong lg_weight") {
695
+ for (uint8_t weight: {1, 70}) {
696
+ auto bytes = estimation.serialize();
697
+ put_value(bytes, estimation_compactor + lg_weight, weight);
698
+ check_rejected(bytes);
699
+ }
700
+ auto bytes = exact.serialize();
701
+ put_value(bytes, exact_compactor + lg_weight, static_cast<uint8_t>(1));
702
+ check_rejected(bytes);
703
+ // the second compactor repeats the first one's lg_weight
704
+ bytes = estimation.serialize();
705
+ uint32_t count;
706
+ std::memcpy(&count, bytes.data() + estimation_compactor + num_items, sizeof(count));
707
+ const size_t second_compactor = estimation_compactor + items + count * sizeof(float);
708
+ REQUIRE(bytes[second_compactor + lg_weight] == 1);
709
+ put_value(bytes, second_compactor + lg_weight, static_cast<uint8_t>(0));
710
+ check_rejected(bytes);
711
+ }
712
+
713
+ SECTION("no levels") {
714
+ auto bytes = exact.serialize();
715
+ bytes[num_levels_offset] = 0;
716
+ check_rejected(bytes);
717
+ bytes = raw.serialize();
718
+ bytes[num_levels_offset] = 0;
719
+ check_rejected(bytes);
720
+ }
721
+
722
+ SECTION("no items") {
723
+ auto bytes = exact.serialize();
724
+ put_value(bytes, exact_compactor + num_items, static_cast<uint32_t>(0));
725
+ check_rejected(bytes);
726
+ bytes = raw.serialize();
727
+ bytes[num_raw_items_offset] = 0;
728
+ check_rejected(bytes);
729
+ }
730
+
731
+ SECTION("compactor header beyond the input") {
732
+ auto bytes = estimation.serialize();
733
+ bytes.resize(estimation_compactor + 10);
734
+ REQUIRE_THROWS_AS(req_sketch<float>::deserialize(bytes.data(), bytes.size()), std::out_of_range);
735
+ }
736
+ }
737
+
562
738
  } /* namespace datasketches */
@@ -202,7 +202,7 @@ void ebpps_sketch<T, A>::merge(const ebpps_sketch<T, A>& sk) {
202
202
  // need to swap this with sk to merge, so make a copy, swap,
203
203
  // and use that to merge
204
204
  ebpps_sketch sk_copy(sk);
205
- swap(*this, sk_copy);
205
+ std::swap(*this, sk_copy);
206
206
  internal_merge(sk_copy);
207
207
  } else {
208
208
  internal_merge(sk);
@@ -25,6 +25,26 @@
25
25
 
26
26
  #include <sstream>
27
27
 
28
+ // An allocator outside the datasketches namespace, as an application's would be: argument-dependent
29
+ // lookup finds no swap() in its namespace.
30
+ namespace ebpps_test_alloc {
31
+
32
+ template<typename T>
33
+ struct plain_allocator {
34
+ using value_type = T;
35
+ plain_allocator() = default;
36
+ template<typename U>
37
+ plain_allocator(const plain_allocator<U>&) {}
38
+ T* allocate(std::size_t n) { return static_cast<T*>(::operator new(n * sizeof(T))); }
39
+ void deallocate(T* p, std::size_t) { ::operator delete(p); }
40
+ };
41
+ template<typename T, typename U>
42
+ bool operator==(const plain_allocator<T>&, const plain_allocator<U>&) { return true; }
43
+ template<typename T, typename U>
44
+ bool operator!=(const plain_allocator<T>&, const plain_allocator<U>&) { return false; }
45
+
46
+ } // namespace ebpps_test_alloc
47
+
28
48
  namespace datasketches {
29
49
 
30
50
  using ebpps_test_sketch = ebpps_sketch<test_type, test_allocator<test_type>>;
@@ -93,4 +113,17 @@ TEST_CASE( "ebpps merge", "[ebpps_sketch][test_type]") {
93
113
  REQUIRE(test_allocator_net_allocations == 0);
94
114
  }
95
115
 
116
+ TEST_CASE("ebpps merge heavier sketch by const reference with an application allocator", "[ebpps_sketch]") {
117
+ using sketch = ebpps_sketch<int, ebpps_test_alloc::plain_allocator<int>>;
118
+ sketch light(5);
119
+ sketch heavy(5);
120
+ for (int i = 0; i < 10; ++i) light.update(i);
121
+ for (int i = 0; i < 100; ++i) heavy.update(i, 2.0);
122
+ const sketch& heavy_ref = heavy;
123
+ light.merge(heavy_ref);
124
+ REQUIRE(light.get_n() == 110);
125
+ REQUIRE(light.get_cumulative_weight() == Approx(210.0));
126
+ REQUIRE(heavy.get_n() == 100);
96
127
  }
128
+
129
+ } // namespace datasketches
@@ -378,7 +378,7 @@ size_t tdigest<T, A>::get_serialized_size_bytes(bool with_buffer) const {
378
378
  template<typename T, typename A>
379
379
  auto tdigest<T, A>::serialize(unsigned header_size_bytes, bool with_buffer) const -> vector_bytes {
380
380
  if (!with_buffer) const_cast<tdigest*>(this)->compress(); // side effect
381
- vector_bytes bytes(get_serialized_size_bytes(with_buffer), 0, buffer_.get_allocator());
381
+ vector_bytes bytes(header_size_bytes + get_serialized_size_bytes(with_buffer), 0, buffer_.get_allocator());
382
382
  uint8_t* ptr = bytes.data() + header_size_bytes;
383
383
  *ptr++ = get_preamble_longs();
384
384
  *ptr++ = SERIAL_VERSION;
@@ -18,6 +18,7 @@
18
18
  */
19
19
 
20
20
  #include <catch2/catch.hpp>
21
+ #include <algorithm>
21
22
  #include <cmath>
22
23
  #include <cstdint>
23
24
  #include <cstring>
@@ -347,6 +348,27 @@ TEST_CASE("serialize deserialize bytes many values with buffer", "[tdigest]") {
347
348
  REQUIRE(td.get_quantile(0.5) == deserialized_td.get_quantile(0.5));
348
349
  }
349
350
 
351
+ TEST_CASE("serialize bytes with header", "[tdigest]") {
352
+ const unsigned header_size_bytes = 7;
353
+ auto check = [header_size_bytes](const tdigest<double>& td, bool with_buffer) {
354
+ auto bytes = td.serialize(header_size_bytes, with_buffer);
355
+ auto bytes_no_header = td.serialize(0, with_buffer);
356
+ REQUIRE(bytes.size() == header_size_bytes + bytes_no_header.size());
357
+ REQUIRE(bytes.size() == header_size_bytes + td.get_serialized_size_bytes(with_buffer));
358
+ REQUIRE(std::equal(bytes_no_header.begin(), bytes_no_header.end(), bytes.begin() + header_size_bytes));
359
+ auto deserialized_td = tdigest<double>::deserialize(bytes.data() + header_size_bytes, bytes.size() - header_size_bytes);
360
+ REQUIRE(td.get_total_weight() == deserialized_td.get_total_weight());
361
+ };
362
+ tdigest<double> td(100);
363
+ check(td, false);
364
+ td.update(123);
365
+ check(td, false);
366
+ check(td, true);
367
+ for (int i = 0; i < 10000; ++i) td.update(i);
368
+ check(td, true);
369
+ check(td, false);
370
+ }
371
+
350
372
  TEST_CASE("serialize deserialize steam and bytes equivalence empty", "[tdigest]") {
351
373
  tdigest<double> td(100);
352
374
  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
@@ -48,6 +48,15 @@ public:
48
48
  */
49
49
  explicit array_tuple_intersection(uint64_t seed = DEFAULT_SEED, const Policy& policy = Policy(), const Allocator& allocator = Allocator());
50
50
 
51
+ /**
52
+ * Updates the intersection with a given sketch.
53
+ * The sketch must have the same number of values as the intersection policy.
54
+ * @param sketch to intersect with
55
+ * @throw std::invalid_argument if the number of values does not match
56
+ */
57
+ template<typename FwdSketch>
58
+ void update(FwdSketch&& sketch);
59
+
51
60
  /**
52
61
  * Produces a copy of the current state of the intersection.
53
62
  * If update() was not called, the state is the infinite "universe",
@@ -17,12 +17,26 @@
17
17
  * under the License.
18
18
  */
19
19
 
20
+ #include <stdexcept>
21
+ #include <string>
22
+
20
23
  namespace datasketches {
21
24
 
22
25
  template<typename Array, typename Policy, typename Allocator>
23
26
  array_tuple_intersection<Array, Policy, Allocator>::array_tuple_intersection(uint64_t seed, const Policy& policy, const Allocator& allocator):
24
27
  Base(seed, policy, allocator) {}
25
28
 
29
+ template<typename Array, typename Policy, typename Allocator>
30
+ template<typename FwdSketch>
31
+ void array_tuple_intersection<Array, Policy, Allocator>::update(FwdSketch&& sketch) {
32
+ const uint8_t num_values = this->state_.get_policy().get_external_policy().get_num_values();
33
+ if (sketch.get_num_values() != num_values) {
34
+ throw std::invalid_argument("number of values mismatch: intersection has " + std::to_string(num_values)
35
+ + ", sketch has " + std::to_string(sketch.get_num_values()));
36
+ }
37
+ Base::update(std::forward<FwdSketch>(sketch));
38
+ }
39
+
26
40
  template<typename Array, typename Policy, typename Allocator>
27
41
  auto array_tuple_intersection<Array, Policy, Allocator>::get_result(bool ordered) const -> CompactSketch {
28
42
  return CompactSketch(this->state_.get_policy().get_external_policy().get_num_values(), Base::get_result(ordered));
@@ -60,6 +60,15 @@ public:
60
60
 
61
61
  class builder;
62
62
 
63
+ /**
64
+ * Update the union with a given sketch.
65
+ * The sketch must have the same number of values as the union policy.
66
+ * @param sketch to update the union with
67
+ * @throw std::invalid_argument if the number of values does not match
68
+ */
69
+ template<typename FwdSketch>
70
+ void update(FwdSketch&& sketch);
71
+
63
72
  CompactSketch get_result(bool ordered = true) const;
64
73
 
65
74
  private:
@@ -17,6 +17,9 @@
17
17
  * under the License.
18
18
  */
19
19
 
20
+ #include <stdexcept>
21
+ #include <string>
22
+
20
23
  namespace datasketches {
21
24
 
22
25
  template<typename Array, typename Policy, typename Allocator>
@@ -24,6 +27,17 @@ array_tuple_union<Array, Policy, Allocator>::array_tuple_union(uint8_t lg_cur_si
24
27
  Base(lg_cur_size, lg_nom_size, rf, p, theta, seed, policy, allocator)
25
28
  {}
26
29
 
30
+ template<typename Array, typename Policy, typename Allocator>
31
+ template<typename FwdSketch>
32
+ void array_tuple_union<Array, Policy, Allocator>::update(FwdSketch&& sketch) {
33
+ const uint8_t num_values = this->state_.get_policy().get_external_policy().get_num_values();
34
+ if (sketch.get_num_values() != num_values) {
35
+ throw std::invalid_argument("number of values mismatch: union has " + std::to_string(num_values)
36
+ + ", sketch has " + std::to_string(sketch.get_num_values()));
37
+ }
38
+ Base::update(std::forward<FwdSketch>(sketch));
39
+ }
40
+
27
41
  template<typename Array, typename Policy, typename Allocator>
28
42
  auto array_tuple_union<Array, Policy, Allocator>::get_result(bool ordered) const -> CompactSketch {
29
43
  return CompactSketch(this->state_.get_policy().get_external_policy().get_num_values(), Base::get_result(ordered));
@@ -173,6 +173,53 @@ TEST_CASE("aod intersection: half overlap", "[tuple_sketch]") {
173
173
  REQUIRE(result.get_estimate() == Approx(500).margin(0.01));
174
174
  }
175
175
 
176
+ TEST_CASE("aod union: number of values", "[tuple_sketch]") {
177
+ auto sketch1 = update_array_of_doubles_sketch::builder(1).build();
178
+ sketch1.update(1, std::vector<double>{1});
179
+ auto sketch3 = update_array_of_doubles_sketch::builder(3).build();
180
+ sketch3.update(1, std::vector<double>{1, 2, 3});
181
+ sketch3.update(2, std::vector<double>{4, 5, 6});
182
+
183
+ // matching number of values: entries for the same key are summed
184
+ auto u = array_of_doubles_union::builder(3).build();
185
+ u.update(sketch3);
186
+ u.update(sketch3.compact());
187
+ auto result = u.get_result();
188
+ REQUIRE(result.get_num_values() == 3);
189
+ REQUIRE(result.get_num_retained() == 2);
190
+ for (const auto& entry: result) {
191
+ REQUIRE(entry.second.size() == 3);
192
+ }
193
+
194
+ // mismatch is rejected whether or not keys overlap, and for update and compact sketches
195
+ REQUIRE_THROWS_AS(u.update(sketch1), std::invalid_argument);
196
+ REQUIRE_THROWS_AS(u.update(sketch1.compact()), std::invalid_argument);
197
+ auto u1 = array_of_doubles_union::builder(1).build();
198
+ REQUIRE_THROWS_AS(u1.update(sketch3), std::invalid_argument);
199
+ auto empty3 = update_array_of_doubles_sketch::builder(3).build();
200
+ REQUIRE_THROWS_AS(u1.update(empty3), std::invalid_argument);
201
+
202
+ // a rejected update leaves the union usable
203
+ u1.update(sketch1);
204
+ REQUIRE(u1.get_result().get_num_retained() == 1);
205
+ }
206
+
207
+ TEST_CASE("aod intersection: number of values", "[tuple_sketch]") {
208
+ auto sketch1 = update_array_of_doubles_sketch::builder(1).build();
209
+ sketch1.update(1, std::vector<double>{1});
210
+ auto sketch3 = update_array_of_doubles_sketch::builder(3).build();
211
+ sketch3.update(1, std::vector<double>{1, 2, 3});
212
+
213
+ array_of_doubles_intersection<default_array_of_doubles_union_policy> intersection(DEFAULT_SEED, 3);
214
+ REQUIRE_THROWS_AS(intersection.update(sketch1), std::invalid_argument);
215
+ intersection.update(sketch3);
216
+ REQUIRE_THROWS_AS(intersection.update(sketch1.compact()), std::invalid_argument);
217
+ intersection.update(sketch3.compact());
218
+ auto result = intersection.get_result();
219
+ REQUIRE(result.get_num_values() == 3);
220
+ REQUIRE(result.get_num_retained() == 1);
221
+ }
222
+
176
223
  TEST_CASE("aod a-not-b: half overlap", "[tuple_sketch]") {
177
224
  double a[1] = {1};
178
225
 
@@ -1 +1 @@
1
- 5.3.0
1
+ 5.3.1
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: datasketches
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.3
4
+ version: 0.5.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane